Get files to be packed via npm pack --dry-run --json (#682)

Author: tommy-mitchell

package.json

@@ -14,7 +14,7 @@
 		"yarn": ">=1.7.0"
 	},
 	"scripts": {
-		"test": "xo && ava && ava test/integration.js --no-worker-threads"
+		"test": "xo && ava"
 	},
 	"files": [
 		"source"
@@ -60,6 +60,7 @@
 		"ow": "^1.1.1",
 		"p-memoize": "^7.1.1",
 		"p-timeout": "^6.1.1",
+		"path-exists": "^5.0.0",
 		"pkg-dir": "^7.0.0",
 		"read-pkg-up": "^9.1.0",
 		"rxjs": "^7.8.0",
@@ -71,18 +72,17 @@
 	"devDependencies": {
 		"ava": "^5.2.0",
 		"common-tags": "^1.8.2",
-		"esmock": "^2.2.0",
+		"esmock": "^2.2.1",
 		"fs-extra": "^11.1.1",
 		"sinon": "^15.0.3",
+		"tempy": "^3.0.0",
+		"write-pkg": "^5.1.0",
 		"xo": "^0.53.1"
 	},
 	"ava": {
 		"environmentVariables": {
 			"FORCE_HYPERLINK": "1"
 		},
-		"files": [
-			"!test/integration.js"
-		],
 		"nodeArguments": [
 			"--loader=esmock"
 		]

source/git-util.js

@@ -2,7 +2,7 @@ import path from 'node:path';
 import {execa} from 'execa';
 import escapeStringRegexp from 'escape-string-regexp';
 import ignoreWalker from 'ignore-walk';
-import {packageDirectorySync} from 'pkg-dir';
+import {packageDirectory} from 'pkg-dir';
 import Version from './version.js';
 
 export const latestTag = async () => {
@@ -27,7 +27,7 @@ export const newFilesSinceLastRelease = async () => {
 	} catch {
 		// Get all files under version control
 		return ignoreWalker({
-			path: packageDirectorySync(),
+			path: await packageDirectory(),
 			ignoreFiles: ['.gitignore'],
 		});
 	}

source/npm/util.js

@@ -1,39 +1,14 @@
-import fs from 'node:fs';
 import path from 'node:path';
+import {pathExists} from 'path-exists';
 import {execa} from 'execa';
 import pTimeout from 'p-timeout';
 import ow from 'ow';
 import npmName from 'npm-name';
 import chalk from 'chalk';
-import {packageDirectorySync} from 'pkg-dir';
-import ignoreWalker from 'ignore-walk';
-import minimatch from 'minimatch';
+import {packageDirectory} from 'pkg-dir';
 import semver from 'semver';
 import Version from '../version.js';
 
-// According to https://docs.npmjs.com/files/package.json#files
-// npm's default behavior is to ignore these files.
-const filesIgnoredByDefault = [
-	'.*.swp',
-	'.npmignore',
-	'.gitignore',
-	'._*',
-	'.DS_Store',
-	'.hg',
-	'.npmrc',
-	'.lock-wscript',
-	'.svn',
-	'.wafpickle-N',
-	'*.orig',
-	'config.gypi',
-	'CVS',
-	'node_modules/**/*',
-	'npm-debug.log',
-	'package-lock.json',
-	'.git/**/*',
-	'.git',
-];
-
 export const checkConnection = () => pTimeout(
 	(async () => {
 		try {
@@ -154,137 +129,24 @@ export const verifyRecentNpmVersion = async () => {
 	Version.verifyRequirementSatisfied('npm', npmVersion);
 };
 
-export const checkIgnoreStrategy = ({files}) => {
-	if (!files && !npmignoreExistsInPackageRootDir()) {
+const npmignoreExistsInPackageRootDir = async () => {
+	const rootDir = await packageDirectory();
+	return pathExists(path.resolve(rootDir, '.npmignore'));
+};
+
+export const checkIgnoreStrategy = async ({files}) => {
+	if (!files && !(await npmignoreExistsInPackageRootDir())) {
 		console.log(`
 		\n${chalk.bold.yellow('Warning:')} No ${chalk.bold.cyan('files')} field specified in ${chalk.bold.magenta('package.json')} nor is a ${chalk.bold.magenta('.npmignore')} file present. Having one of those will prevent you from accidentally publishing development-specific files along with your package's source code to npm.
 		`);
 	}
 };
 
-function npmignoreExistsInPackageRootDir() {
-	const rootDir = packageDirectorySync();
-	return fs.existsSync(path.resolve(rootDir, '.npmignore'));
-}
-
-function excludeGitAndNodeModulesPaths(singlePath) {
-	return !singlePath.startsWith('.git/') && !singlePath.startsWith('node_modules/');
-}
-
-async function getFilesIgnoredByDotnpmignore(pkg, fileList) {
-	let allowList = await ignoreWalker({
-		path: packageDirectorySync(),
-		ignoreFiles: ['.npmignore'],
-	});
-	allowList = allowList.filter(singlePath => excludeGitAndNodeModulesPaths(singlePath));
-	return fileList.filter(minimatch.filter(getIgnoredFilesGlob(allowList, pkg.directories), {matchBase: true, dot: true}));
-}
-
-function filterFileList(globArray, fileList) {
-	if (globArray.length === 0) {
-		return [];
-	}
-
-	const globString = globArray.length > 1 ? `{${globArray.filter(singlePath => excludeGitAndNodeModulesPaths(singlePath))}}` : globArray[0];
-	return fileList.filter(minimatch.filter(globString, {matchBase: true, dot: true})); // eslint-disable-line unicorn/no-array-callback-reference, unicorn/no-array-method-this-argument
-}
-
-async function getFilesIncludedByDotnpmignore(pkg, fileList) {
-	const allowList = await ignoreWalker({
-		path: packageDirectorySync(),
-		ignoreFiles: ['.npmignore'],
-	});
-	return filterFileList(allowList, fileList);
-}
-
-function getFilesNotIncludedInFilesProperty(pkg, fileList) {
-	const globArrayForFilesAndDirectories = [...pkg.files];
-	const rootDir = packageDirectorySync();
-	for (const glob of pkg.files) {
-		try {
-			if (fs.statSync(path.resolve(rootDir, glob)).isDirectory()) {
-				globArrayForFilesAndDirectories.push(`${glob}/**/*`);
-			}
-		} catch {}
-	}
-
-	const result = fileList.filter(minimatch.filter(getIgnoredFilesGlob(globArrayForFilesAndDirectories, pkg.directories), {matchBase: true, dot: true}));
-	return result.filter(minimatch.filter(getDefaultIncludedFilesGlob(pkg.main), {nocase: true, matchBase: true}));
-}
-
-function getFilesIncludedInFilesProperty(pkg, fileList) {
-	const globArrayForFilesAndDirectories = [...pkg.files];
-	const rootDir = packageDirectorySync();
-	for (const glob of pkg.files) {
-		try {
-			if (fs.statSync(path.resolve(rootDir, glob)).isDirectory()) {
-				globArrayForFilesAndDirectories.push(`${glob}/**/*`);
-			}
-		} catch {}
-	}
-
-	return filterFileList(globArrayForFilesAndDirectories, fileList);
-}
-
-function getDefaultIncludedFilesGlob(mainFile) {
-	// According to https://docs.npmjs.com/files/package.json#files
-	// npm's default behavior is to always include these files.
-	const filesAlwaysIncluded = [
-		'package.json',
-		'README*',
-		'CHANGES*',
-		'CHANGELOG*',
-		'HISTORY*',
-		'LICENSE*',
-		'LICENCE*',
-		'NOTICE*',
-	];
-	if (mainFile) {
-		filesAlwaysIncluded.push(mainFile);
-	}
-
-	return `!{${filesAlwaysIncluded}}`;
-}
-
-function getIgnoredFilesGlob(globArrayFromFilesProperty, packageDirectories) {
-	// Test files are assumed not to be part of the package
-	let testDirectoriesGlob = '';
-	if (packageDirectories && Array.isArray(packageDirectories.test)) {
-		testDirectoriesGlob = packageDirectories.test.join(',');
-	} else if (packageDirectories && typeof packageDirectories.test === 'string') {
-		testDirectoriesGlob = packageDirectories.test;
-	} else {
-		// Fallback to `test` directory
-		testDirectoriesGlob = 'test/**/*';
-	}
-
-	return `!{${globArrayFromFilesProperty.join(',')},${filesIgnoredByDefault.join(',')},${testDirectoriesGlob}}`;
-}
-
-// Get all files which will be ignored by either `.npmignore` or the `files` property in `package.json` (if defined).
-export const getNewAndUnpublishedFiles = async (pkg, newFiles = []) => {
-	if (pkg.files) {
-		return getFilesNotIncludedInFilesProperty(pkg, newFiles);
-	}
-
-	if (npmignoreExistsInPackageRootDir()) {
-		return getFilesIgnoredByDotnpmignore(pkg, newFiles);
-	}
-
-	return [];
-};
-
-export const getFirstTimePublishedFiles = async (pkg, newFiles = []) => {
-	let result;
-	if (pkg.files) {
-		result = getFilesIncludedInFilesProperty(pkg, newFiles);
-	} else if (npmignoreExistsInPackageRootDir()) {
-		result = await getFilesIncludedByDotnpmignore(pkg, newFiles);
-	} else {
-		result = newFiles;
-	}
+export const getFilesToBePacked = async () => {
+	const {stdout} = await execa('npm', ['pack', '--dry-run', '--json'], {cwd: await packageDirectory()});
 
-	return result.filter(minimatch.filter(`!{${filesIgnoredByDefault}}`, {matchBase: true, dot: true})).filter(minimatch.filter(getDefaultIncludedFilesGlob(pkg.main), {nocase: true, matchBase: true}));
+	const {files} = JSON.parse(stdout).at(0);
+	return files.map(file => file.path);
 };
 
 export const getRegistryUrl = async (pkgManager, pkg) => {

source/ui.js

@@ -6,7 +6,7 @@ import isScoped from 'is-scoped';
 import isInteractive from 'is-interactive';
 import * as util from './util.js';
 import * as git from './git-util.js';
-import {prereleaseTags, checkIgnoreStrategy, getRegistryUrl, isExternalRegistry} from './npm/util.js';
+import * as npmUtil from './npm/util.js';
 import Version from './version.js';
 import prettyVersionDiff from './pretty-version-diff.js';
 
@@ -126,11 +126,11 @@ const ui = async (options, {pkg, pkgPath}) => {
 	const extraBaseUrls = ['gitlab.com'];
 	const repoUrl = pkg.repository && githubUrlFromGit(pkg.repository.url, {extraBaseUrls});
 	const pkgManager = options.yarn ? 'yarn' : 'npm';
-	const registryUrl = await getRegistryUrl(pkgManager, pkg);
+	const registryUrl = await npmUtil.getRegistryUrl(pkgManager, pkg);
 	const releaseBranch = options.branch;
 
 	if (options.runPublish) {
-		checkIgnoreStrategy(pkg);
+		await npmUtil.checkIgnoreStrategy(pkg);
 
 		const answerIgnoredFiles = await checkNewFilesAndDependencies(pkg, pkgPath);
 		if (!answerIgnoredFiles) {
@@ -253,7 +253,7 @@ const ui = async (options, {pkg, pkgPath}) => {
 			message: 'How should this pre-release version be tagged in npm?',
 			when: answers => options.runPublish && (Version.isPrereleaseOrIncrement(answers.customVersion) || Version.isPrereleaseOrIncrement(answers.version)) && !options.tag,
 			async choices() {
-				const existingPrereleaseTags = await prereleaseTags(pkg.name);
+				const existingPrereleaseTags = await npmUtil.prereleaseTags(pkg.name);
 
 				return [
 					...existingPrereleaseTags,
@@ -283,7 +283,7 @@ const ui = async (options, {pkg, pkgPath}) => {
 		},
 		publishScoped: {
 			type: 'confirm',
-			when: isScoped(pkg.name) && options.availability.isAvailable && !options.availability.isUnknown && options.runPublish && (pkg.publishConfig && pkg.publishConfig.access !== 'restricted') && !isExternalRegistry(pkg),
+			when: isScoped(pkg.name) && options.availability.isAvailable && !options.availability.isUnknown && options.runPublish && (pkg.publishConfig && pkg.publishConfig.access !== 'restricted') && !npmUtil.isExternalRegistry(pkg),
 			message: `This scoped repo ${chalk.bold.magenta(pkg.name)} hasn't been published. Do you want to publish it publicly?`,
 			default: false,
 		},

source/util.js

@@ -72,9 +72,14 @@ export const getTagVersionPrefix = pMemoize(async options => {
 
 export const joinList = list => chalk.reset(list.map(item => `- ${item}`).join('\n'));
 
-export const getNewFiles = async pkg => {
+export const getNewFiles = async () => {
 	const listNewFiles = await gitUtil.newFilesSinceLastRelease();
-	return {unpublished: await npmUtil.getNewAndUnpublishedFiles(pkg, listNewFiles), firstTime: await npmUtil.getFirstTimePublishedFiles(pkg, listNewFiles)};
+	const listPkgFiles = await npmUtil.getFilesToBePacked();
+
+	return {
+		unpublished: listNewFiles.filter(file => !listPkgFiles.includes(file) && !file.startsWith('.git')),
+		firstTime: listNewFiles.filter(file => listPkgFiles.includes(file)),
+	};
 };
 
 export const getNewDependencies = async (newPkg, pkgPath) => {

test/_utils.js

@@ -48,3 +48,9 @@ export const assertTaskDisabled = (t, taskTitle) => {
 export const assertTaskDoesntExist = (t, taskTitle) => {
 	t.true(SilentRenderer.tasks.every(task => task.title !== taskTitle), `'${taskTitle}' exists!`);
 };
+
+export const runIfExists = async (func, ...args) => {
+	if (typeof func === 'function') {
+		await func(...args);
+	}
+};

test/fixtures/files/dot-github/.github/pull_request_template.md

@@ -0,0 +1,9 @@
+<!--
+
+Thanks for submitting a pull request 🙌
+
+**Note:** Please don't create a pull request which has significant changes (i.e. adds new functionality or modifies existing one in a non-trivial way) without creating an issue first.
+
+Try to limit the scope of your pull request and provide a general description of the changes. If this fixes an open issue, link to it in the following way: `Fixes #321`. New features and bug fixes should come with tests.
+
+-->

test/fixtures/files/dot-github/index.js

@@ -0,0 +1 @@
+console.log('foo');

test/fixtures/files/dot-github/package.json

@@ -0,0 +1,5 @@
+{
+	"name": "foo",
+	"version": "0.0.0",
+	"files": ["index.js"]
+}

test/fixtures/files/files-and-npmignore/package.json

@@ -0,0 +1,5 @@
+{
+	"name": "foo",
+	"version": "0.0.0",
+	"files": ["source"]
+}

test/fixtures/files/files-and-npmignore/readme.md

@@ -0,0 +1 @@
+# Foo

test/fixtures/files/files-and-npmignore/source/.npmignore

@@ -0,0 +1 @@
+index.test-d.ts

test/fixtures/files/files-and-npmignore/source/bar.js

@@ -0,0 +1 @@
+console.log('bar');

test/fixtures/files/files-and-npmignore/source/foo.js

@@ -0,0 +1 @@
+console.log('foo');

test/fixtures/files/files-and-npmignore/source/index.d.ts

@@ -0,0 +1,2 @@
+export function foo(): string;
+export function bar(): string;

test/fixtures/files/files-and-npmignore/source/index.test-d.ts

@@ -0,0 +1,5 @@
+import {expectType} from 'tsd';
+import {foo, bar} from '.';
+
+expectType<string>(foo());
+expectType<string>(bar());

test/fixtures/files/files-slash/index.js

@@ -0,0 +1 @@
+console.log('foo');

test/fixtures/files/files-slash/package.json

@@ -0,0 +1,5 @@
+{
+	"name": "foo",
+	"version": "0.0.0",
+	"files": ["/index.js"]
+}

test/fixtures/files/gitignore/dist/index.js

@@ -0,0 +1,2 @@
+console.log('foo');
+console.log('bar');

test/fixtures/files/gitignore/gitignore

@@ -0,0 +1,3 @@
+# This file is renamed to `.gitignore` in the test
+# This is not named `.gitignore` to allow `dist/` to be committed
+dist

test/fixtures/files/gitignore/index.d.ts

@@ -0,0 +1 @@
+export default function foo(): string;

test/fixtures/files/gitignore/index.js

@@ -0,0 +1,3 @@
+export default function foo() {
+	return 'bar';
+}

test/fixtures/files/gitignore/index.test-d.ts

@@ -0,0 +1,4 @@
+import {expectType} from 'tsd';
+import foo from '.';
+
+expectType<string>(foo());

test/fixtures/files/gitignore/package.json

@@ -0,0 +1,5 @@
+{
+	"name": "foo",
+	"version": "0.0.0",
+	"files": ["dist"]
+}

test/fixtures/files/gitignore/readme.md

@@ -0,0 +1 @@
+# Foo

test/fixtures/files/has-readme-and-license/index.js

@@ -0,0 +1 @@
+console.log('foo');

test/fixtures/files/has-readme-and-license/license.md

@@ -0,0 +1 @@
+MIT