Add a script to pre-generate changelog entries

I have been using and improving that script for some time now, at least all 2.5 and 2.6 releases. 👌

.eslintrc.yml

@@ -50,7 +50,11 @@ rules:
   semi-style: [error, last]
   semi: [error, always]
   space-before-blocks: error
-  space-before-function-paren: [error, never]
+  space-before-function-paren:
+    - error
+    - anonymous: never
+      named: never
+      asyncArrow: always # Otherwise requires `async()`
   space-in-parens: [error, never]
   space-infix-ops: error
   spaced-comment: [error, always]

package.json

@@ -73,6 +73,7 @@
     "eslint": "4.11.0",
     "font-awesome": "4.7.0",
     "fuzzy": "0.1.3",
+    "graphql-request": "1.4.0",
     "handlebars": "4.0.11",
     "handlebars-loader": "1.6.0",
     "intersection-observer": "0.4.3",

scripts/.eslintrc.yml

@@ -0,0 +1,5 @@
+---
+
+# Necessary to support async/await... grumble grumble...
+parserOptions:
+  ecmaVersion: 8

scripts/changelog.js

@@ -0,0 +1,687 @@
+/*
+This (very The Lounge-custom) script is a helper to generate changelog entries.
+
+Note that it is not meant to produce fully-automated changelogs like other tools
+do, but merely prepare a changelog entry without risks of mistyping a URL or
+missing a contribution: changelogs are meant for humans, and therefore must be
+manually curated as such, with ❤️.
+
+## Set up:
+
+- Create a personal access token with `public_repo` at
+  https://github.com/settings/tokens. Make sure to write it down as you will not
+  be able to display it again.
+
+- Use Node.js v8+:
+
+  ```sh
+  nvm install 8
+  ```
+
+## Usage
+
+npm v5 removes packages not listed in package.json when running `npm install` so
+it is very likely you will have to run all those each time:
+
+```sh
+ export CHANGELOG_TOKEN=<The personal access token created on GitHub above>
+node scripts/changelog <version>
+```
+
+`<version>` must *not* be prefixed with `v`. It is formatted either:
+
+- `MAJOR.MINOR.PATCH` for a stable release, for example `2.5.0`
+- `MAJOR.MINOR.PATCH-(pre|rc).N` for a pre-release, for example `2.5.0-rc.1`
+
+## TODOs:
+
+- Use better labels for better categorization
+- Add some stats to the git commit (how many LOCs total / in this release, etc.)
+- This script requires Node v8, but `npm version` currently fails with Node v8
+  as we gitignore package-lock.json (how is that even a thing?!).
+*/
+
+"use strict";
+
+const _ = require("lodash");
+const colors = require("colors/safe");
+const fs = require("fs");
+const GraphQLClient = require("graphql-request").GraphQLClient;
+const moment = require("moment");
+const util = require("util");
+const log = require("../src/log");
+const packageJson = require("../package.json");
+const token = process.env.CHANGELOG_TOKEN;
+
+const readFile = util.promisify(fs.readFile);
+const writeFile = util.promisify(fs.writeFile);
+
+// CLI argument validations
+
+if (token === undefined) {
+	log.error(`Environment variable ${colors.bold("CHANGELOG_TOKEN")} must be set.`);
+	process.exit(1);
+}
+
+if (process.argv[2] === undefined) {
+	log.error(`Argument ${colors.bold("version")} is missing`);
+	process.exit(1);
+}
+
+const version = process.argv[2];
+
+if (!/^[0-9]+\.[0-9]+\.[0-9]+(-(pre|rc)+\.[0-9]+)?$/.test(version)) {
+	log.error(`Argument ${colors.bold("version")} is incorrect.`);
+	log.error(`It must match format ${colors.green("x.y.z")} (stable) or ${colors.green("x.y.z-(pre|rc).n")} (pre-release).`);
+	process.exit(1);
+}
+
+// Templates
+
+function prereleaseTemplate(items) {
+	return `
+## v${items.version} - ${items.date} [Pre-release]
+
+[See the full changelog](${items.fullChangelogUrl})
+
+This is a release candidate for v${stableVersion(items.version)} to ensure maximum stability for public release.
+Please refer to the commit list given above for a complete list of changes, or wait for the stable release to get a thoroughly prepared change log entry.
+
+As with all pre-releases, this version requires explicit use of the \`next\` tag to be installed:
+
+\`\`\`sh
+npm install -g thelounge@next
+\`\`\`
+`;
+}
+
+function stableTemplate(items) {
+	return `
+## v${items.version} - ${items.date}
+
+For more details, [see the full changelog](${items.fullChangelogUrl}) and [milestone](${items.milestone.url}?closed=1).
+
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+@@ DESCRIPTION, ANNOUNCEMENT, ETC. @@
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+
+### Added
+
+### Changed
+
+${_.isEmpty(items.dependencies) ? "" :
+		`- Update production dependencies to their latest versions, by [Greenkeeper](https://greenkeeper.io/) 🚀:
+${printDependencyList(items.dependencies)}`
+}
+
+### Deprecated
+
+${printList(items.deprecations)}
+
+### Removed
+
+### Fixed
+
+### Security
+
+${printList(items.security)}
+
+### Documentation
+
+${_.isEmpty(items.documentation) ? "" :
+		`In the main repository:
+
+${printList(items.documentation)}`
+}
+
+${_.isEmpty(items.websiteDocumentation) ? "" :
+		`On the [website repository](https://github.com/thelounge/thelounge.github.io):
+
+${printList(items.websiteDocumentation)}`
+}
+
+### Internals
+
+${printList(items.internals)}${
+	_.isEmpty(items.devDependencies) ? "" : `
+- Update development dependencies to their latest versions, by [Greenkeeper](https://greenkeeper.io/) 🚀:
+${printDependencyList(items.devDependencies)}`}
+
+@@@@@@@@@@@@@@@@@@@
+@@ UNCATEGORIZED @@
+@@@@@@@@@@@@@@@@@@@
+
+${printList(items.uncategorized)}
+`;
+}
+
+// Returns true if the given version is a pre-release (i.e. 2.0.0-pre.3,
+// 2.5.0-rc.1, etc.), or false otherwise
+function isPrerelease(v) {
+	return v.includes("-");
+}
+
+// Returns the stable version that this pre-release version is targeting. For
+// example, if new version is 2.5.0-rc.2, next stable version will be 2.5.0.
+function stableVersion(prereleaseVersion) {
+	return prereleaseVersion.substr(0, prereleaseVersion.indexOf("-"));
+}
+
+// Generates a compare-view URL between 2 versions of The Lounge
+function fullChangelogUrl(v1, v2) {
+	return `https://github.com/thelounge/lounge/compare/v${v1}...v${v2}`;
+}
+
+// This class is a facade to fetching details about commits / PRs / tags / etc.
+// for a given repository of our organization.
+class RepositoryFetcher {
+	// Holds a GraphQLClient and the name of the repository within the
+	// organization https://github.com/thelounge.
+	constructor(graphqlClient, repositoryName) {
+		this.graphqlClient = graphqlClient;
+		this.repositoryName = repositoryName;
+	}
+
+	// Base function that actually makes the GraphQL API call
+	async fetch(query, variables = {}) {
+		return this.graphqlClient.request(
+			query,
+			Object.assign(variables, {repositoryName: this.repositoryName})
+		);
+	}
+
+	// Returns the git commit that is attached to a given tag
+	async fetchTaggedCommit(tag) {
+		const tagQuery = `query fetchTaggedCommit($repositoryName: String!, $tag: String!) {
+			repository(owner: "thelounge", name: $repositoryName) {
+				ref(qualifiedName: $tag) {
+					tag: target {
+						... on Tag {
+							commit: target {
+								oid
+							}
+						}
+					}
+				}
+			}
+		}`;
+		const data = await this.fetch(tagQuery, {tag});
+		return data.repository.ref.tag.commit;
+	}
+
+	// Returns an array of annotated commits that have been made on the master
+	// branch since a given version. Each commit is an object that can optionally
+	// have a `pullRequestId` if this is a merge-PR commit.
+	async fetchCommitsSince(stopCommit) {
+		const commitsQuery = `query fetchCommits($repositoryName: String!, $afterCursor: String) {
+			repository(owner: "thelounge", name: $repositoryName) {
+				ref(qualifiedName: "master") {
+					target {
+						... on Commit {
+							history(first: 100, after: $afterCursor) {
+								pageInfo {
+									hasNextPage
+									endCursor
+								}
+								commits: nodes {
+									oid
+									abbreviatedOid
+									messageHeadline
+									url
+									author {
+										user {
+											login
+											url
+										}
+									}
+									comments(first: 100) {
+										nodes {
+											body
+											authorAssociation
+										}
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+		}`;
+
+		// Recursive function that retrieves commits page after page until the last
+		// page or a given commit are reached.
+		const fetchPaginatedCommits = async (afterCursor = null) => {
+			const data = await this.fetch(commitsQuery, {afterCursor});
+			const {commits, pageInfo} = data.repository.ref.target.history;
+
+			if (commits.map(({oid}) => oid).includes(stopCommit.oid)) {
+				return _.takeWhile(commits, ({oid}) => oid !== stopCommit.oid);
+			} else if (pageInfo.hasNextPage) {
+				return commits.concat(await fetchPaginatedCommits(stopCommit, pageInfo.endCursor));
+			}
+
+			return commits;
+		};
+
+		const commits = await fetchPaginatedCommits();
+
+		commits.forEach((commit) => {
+			commit.author = commit.author.user;
+
+			const resultPR = /^Merge pull request #([0-9]+) .+/.exec(commit.messageHeadline);
+
+			if (resultPR) {
+				commit.pullRequestId = parseInt(resultPR[1], 10);
+			}
+		});
+
+		return commits.reverse();
+	}
+
+	// Returns the last version prior to this new one. If new version is stable,
+	// the previous one will be stable as well (all pre-release versions will be
+	// skipped).
+	async fetchPreviousVersion(newVersion) {
+		const lastTagsQuery = `query fetchPreviousVersion($repositoryName: String!) {
+			repository(owner: "thelounge", name: $repositoryName) {
+				refs(refPrefix: "refs/tags/", first: 20, direction: DESC) {
+					tags: nodes {
+						name
+					}
+				}
+			}
+		}`;
+		const data = await this.fetch(lastTagsQuery);
+		const tags = data.repository.refs.tags;
+		let tag;
+
+		if (isPrerelease(newVersion)) {
+			tag = tags[0];
+		} else {
+			tag = tags.find(({name}) => !isPrerelease(name));
+		}
+
+		return tag.name.substr(1);
+	}
+
+	// Returns information on a milestone associated to a version (i.e. not a
+	// tag!) of the repository
+	async fetchMilestone(targetVersion) {
+		const milestonesQuery = `query fetchMilestone($repositoryName: String!) {
+			repository(owner: "thelounge", name: $repositoryName) {
+				milestones(last: 20) {
+					nodes {
+						title
+						url
+					}
+				}
+			}
+		}`;
+		const data = await this.fetch(milestonesQuery);
+		return data.repository.milestones.nodes.find(({title}) => title === targetVersion);
+	}
+
+	// Given a list of PR numbers, retrieve information for all those PRs. They
+	// are returned as a hash whose keys are `PR<number>`.
+	// This is a bit wonky (generating a dynamic GraphQL query) but the GitHub API
+	// does not have a way to retrieve multiple PRs given a list of IDs.
+	async fetchPullRequests(numbers) {
+		if (numbers.length === 0) {
+			return [];
+		}
+
+		const prQuery = `query fetchPullRequests($repositoryName: String!) {
+			repository(owner: "thelounge", name: $repositoryName) {
+				${numbers.map((number) => `
+					PR${number}: pullRequest(number: ${number}) {
+						title
+						url
+						author {
+							login
+							url
+						}
+						labels(first: 20) {
+							nodes {
+								name
+							}
+						}
+						commits(first: 100) {
+							nodes {
+								commit {
+									oid
+								}
+							}
+						}
+					}
+				`).join("")}
+			}
+		}`;
+		const data = await this.fetch(prQuery);
+		return data.repository;
+	}
+
+	// Chain several of the functions above together. Essentially, returns an
+	// array composed of PRs, and commits that belong to no PRs, existing between
+	// a given tag and master.
+	async fetchCommitsAndPullRequestsSince(tag) {
+		const taggedCommit = await this.fetchTaggedCommit(tag);
+		const commits = await this.fetchCommitsSince(taggedCommit);
+		const pullRequestIds = pullRequestNumbersInCommits(commits);
+		const pullRequests = await this.fetchPullRequests(pullRequestIds);
+		return combine(commits, pullRequests);
+	}
+}
+
+// Given an array of annotated commits, returns an array of PR numbers, integers
+function pullRequestNumbersInCommits(commits) {
+	return commits.reduce((array, {pullRequestId}) => {
+		if (pullRequestId) {
+			array.push(pullRequestId);
+		}
+		return array;
+	}, []);
+}
+
+// Given 2 arrays of annotated commits and pull requests, replace merge commits
+// with the pull request information, and remove commits that are already part
+// of a pull request.
+// The goal of this function is to return an array consisting only of pull
+// requests + commits that have been made to `master` directly.
+function combine(allCommits, allPullRequests) {
+	const commitsFromPRs = _.flatMap(allPullRequests, ({commits}) =>
+		commits.nodes.map(({commit}) => commit.oid)
+	);
+
+	return allCommits.reduce((array, commit) => {
+		if (commit.pullRequestId) {
+			const pullRequest = allPullRequests[`PR${commit.pullRequestId}`];
+			pullRequest.number = commit.pullRequestId;
+			array.push(pullRequest);
+		} else if (!commitsFromPRs.includes(commit.oid)) {
+			array.push(commit);
+		}
+
+		return array;
+	}, []);
+}
+
+// Builds a Markdown link for a given pull request object
+function printPullRequestLink({number, url}) {
+	return `[#${number}](${url})`;
+}
+
+// Builds a Markdown link for a given author object
+function printAuthorLink({login, url}) {
+	return `by [@${login}](${url})`;
+}
+
+// Builds a Markdown entry list item depending on its type
+function printLine(entry) {
+	if (entry.title) {
+		return printPullRequest(entry);
+	}
+	return printCommit(entry);
+}
+
+// Builds a Markdown list item for a given pull request
+function printPullRequest(pullRequest) {
+	return `- ${pullRequest.title} (${printPullRequestLink(pullRequest)} ${printAuthorLink(pullRequest.author)})`;
+}
+
+// Builds a Markdown list item for a commit made directly in `master`
+function printCommit({abbreviatedOid, messageHeadline, url, author}) {
+	return `- ${messageHeadline} ([\`${abbreviatedOid}\`](${url}) ${printAuthorLink(author)})`;
+}
+
+// Builds a Markdown list of all given items
+function printList(items) {
+	return items.map((item) => printLine(item)).join("\n");
+}
+
+// Given a "dependencies object" (i.e. keys are package names, values are arrays
+// of pull request numbers), builds a Markdown list of URLs
+function printDependencyList(dependencies) {
+	return _.map(dependencies, (pullRequests, name) =>
+		`  - \`${name}\` (${pullRequests.map(printPullRequestLink).join(", ")})`
+	).join("\n");
+}
+
+const dependencies = Object.keys(packageJson.dependencies);
+const devDependencies = Object.keys(packageJson.devDependencies);
+
+// Returns the package.json section in which that package exists, or undefined
+// if that package is not listed there.
+function whichDependencyType(packageName) {
+	if (dependencies.includes(packageName)) {
+		return "dependencies";
+	} else if (devDependencies.includes(packageName)) {
+		return "devDependencies";
+	}
+}
+
+function hasLabelOrAnnotatedComment({labels, comments}, expected) {
+	return hasLabel(labels, expected) || hasAnnotatedComment(comments, expected);
+}
+
+// Returns true if a label exists amongst a list of labels
+function hasLabel(labels, expected) {
+	return labels && labels.nodes.some(({name}) => name === expected);
+}
+
+function hasAnnotatedComment(comments, expected) {
+	return comments && comments.nodes.some(({authorAssociation, body}) =>
+		["OWNER", "MEMBER"].includes(authorAssociation) &&
+		body.split("\n").includes(`[${expected}]`)
+	);
+}
+
+function isSkipped(entry) {
+	return hasLabelOrAnnotatedComment(entry, "Meta: Skip Changelog");
+}
+
+// Greenkeeper PRs are listed in a special, more concise way in the changelog.
+// Returns true if the PR was open by Greenkeeper, false otherwise.
+function isDependency({author, labels}) {
+	return hasLabel(labels, "greenkeeper") || author.login === "greenkeeper";
+}
+
+function isDocumentation({labels}) {
+	return hasLabel(labels, "Type: Documentation");
+}
+
+function isSecurity({labels}) {
+	return hasLabel(labels, "Type: Security");
+}
+
+function isDeprecation({labels}) {
+	return hasLabel(labels, "Type: Deprecation");
+}
+
+function isInternal(entry) {
+	return hasLabelOrAnnotatedComment(entry, "Meta: Internal");
+}
+
+// Examples:
+//   Update webpack to the latest version
+//   Update `stylelint` to v1.2.3
+//   Update `express` and `ua-parser-js` to latest versions
+//   Update `express`, `chai`, and `ua-parser-js` to ...
+function extractPackages(title) {
+	return /^Update ([\w-,`. ]+) to /.exec(title)[1]
+		.replace(/`/g, "")
+		.split(/, and |, | and /);
+}
+
+// Given an array of entries (PRs or commits), separates them into sections,
+// based on different information that describes them.
+function parse(entries) {
+	return entries.reduce((result, entry) => {
+		if (isSkipped(entry)) {
+			result.skipped.push(entry);
+		} else if (isDependency(entry)) {
+			extractPackages(entry.title).forEach((packageName) => {
+				const dependencyType = whichDependencyType(packageName);
+
+				if (dependencyType) {
+					if (!result[dependencyType][packageName]) {
+						result[dependencyType][packageName] = [];
+					}
+					result[dependencyType][packageName].push(entry);
+				} else {
+					log.info(`${colors.bold(packageName)} was updated in ${colors.green("#" + entry.number)} then removed since last release. Skipping.`);
+				}
+			});
+		} else if (isDocumentation(entry)) {
+			result.documentation.push(entry);
+		} else if (isDeprecation(entry)) {
+			result.deprecations.push(entry);
+		} else if (isSecurity(entry)) {
+			result.security.push(entry);
+		} else if (isInternal(entry)) {
+			result.internals.push(entry);
+		} else {
+			result.uncategorized.push(entry);
+		}
+		return result;
+	}, {
+		skipped: [],
+		dependencies: {},
+		devDependencies: {},
+		deprecations: [],
+		documentation: [],
+		internals: [],
+		security: [],
+		uncategorized: [],
+		unknownDependencies: new Set(),
+	});
+}
+
+// Given a list of entries (pull requests, commits), retrieves GitHub usernames
+// (with format `@username`) of everyone who contributed to this version.
+function extractContributors(entries) {
+	const set = Object.values(entries).reduce((memo, pullRequest) => {
+		if (pullRequest.author.login !== "greenkeeper") {
+			memo.add("@" + pullRequest.author.login);
+		}
+		return memo;
+	}, new Set());
+
+	return Array.from(set)
+		.sort((a, b) => a.localeCompare(b, "en", {sensitivity: "base"}));
+}
+
+const client = new GraphQLClient("https://api.github.com/graphql", {
+	headers: {
+		Authorization: `bearer ${token}`,
+	},
+});
+
+// Main function. Given a version string (i.e. not a tag!), returns a changelog
+// entry and the list of contributors, for both pre-releases and stable
+// releases. Templates are located at the top of this file.
+async function generateChangelogEntry(targetVersion) {
+	let items = {};
+	let template;
+	let contributors = [];
+
+	const codeRepo = new RepositoryFetcher(client, "lounge");
+	const previousVersion = await codeRepo.fetchPreviousVersion(targetVersion);
+
+	if (isPrerelease(targetVersion)) {
+		template = prereleaseTemplate;
+	} else {
+		template = stableTemplate;
+
+		const codeCommitsAndPullRequests = await codeRepo.fetchCommitsAndPullRequestsSince("v" + previousVersion);
+		items = parse(codeCommitsAndPullRequests);
+		items.milestone = await codeRepo.fetchMilestone(targetVersion);
+		contributors = extractContributors(codeCommitsAndPullRequests);
+
+		const websiteRepo = new RepositoryFetcher(client, "thelounge.github.io");
+		items.websiteDocumentation = await websiteRepo.fetchCommitsAndPullRequestsSince("v" + previousVersion);
+	}
+
+	items.version = targetVersion;
+	items.date = moment().format("YYYY-MM-DD");
+	items.fullChangelogUrl = fullChangelogUrl(previousVersion, targetVersion);
+
+	return {
+		changelogEntry: template(items),
+		skipped: items.skipped || [],
+		contributors,
+	};
+}
+
+// Write a changelog entry into the CHANGELOG.md file, right after a marker that
+// indicates where entries are listed.
+async function addToChangelog(newEntry) {
+	const changelogPath = "./CHANGELOG.md";
+	const changelogMarker = "<!-- New entries go after this line -->\n\n";
+
+	const changelog = await readFile(changelogPath, "utf8");
+	const markerPosition = changelog.indexOf(changelogMarker) + changelogMarker.length;
+	const newChangelog =
+		changelog.substring(0, markerPosition) +
+		newEntry +
+		changelog.substring(markerPosition, changelog.length);
+
+	writeFile(changelogPath, newChangelog);
+}
+
+// Wrapping this in an Async IIFE because async/await is only supported within
+// functions. ¯\_(ツ)_/¯
+(async () => {
+	log.info(`Generating a changelog entry for ${colors.bold("v" + version)}, please wait...`);
+	const startTime = Date.now();
+	let changelogEntry, skipped, contributors;
+
+	// Step 1: Generate a changelog entry
+
+	try {
+		({changelogEntry, skipped, contributors} = await generateChangelogEntry(version));
+	} catch (error) {
+		if (error.response && error.response.status === 401) {
+			log.error(`GitHub returned an error: ${colors.red(error.response.message)}`);
+			log.error(`Make sure your personal access token is set with ${colors.bold("public_repo")} scope.`);
+		} else {
+			log.error(error);
+		}
+		process.exit(1);
+	}
+
+	// Step 2: Write that changelog entry into the CHANGELOG.md file
+
+	try {
+		await addToChangelog(`${changelogEntry.trim()}\n\n`);
+	} catch (error) {
+		log.error(error);
+		process.exit(1);
+	}
+
+	log.info(`The generated entry was added at the top of ${colors.bold("CHANGELOG.md")}.`);
+
+	// Step 3 (optional): Print a list of skipped entries if there are any
+	if (skipped.length > 0) {
+		const pad = Math.max(...skipped.map((entry) => (entry.title || entry.messageHeadline).length));
+		log.warn(`${skipped.length} entries were skipped:`);
+		skipped.forEach((entry) => {
+			log.warn(`- ${(entry.title || entry.messageHeadline).padEnd(pad)}  ${colors.gray(entry.url)}`);
+		});
+	}
+
+	// Step 4: Print out some information about what just happened to the console
+	const commitCommand = `git commit -m 'Add changelog entry for v${version}' CHANGELOG.md`;
+	if (isPrerelease(version)) {
+		log.info(`You can now run: ${colors.bold(commitCommand)}`);
+	} else {
+		log.info(`Please edit ${colors.bold("CHANGELOG.md")} to your liking then run: ${colors.bold(commitCommand)}`);
+	}
+
+	log.info(`Finished in ${colors.bold(Date.now() - startTime)}ms.`);
+
+	// Step 5 (optional): Print contributors shout out if it exists
+	if (contributors.length > 0) {
+		log.info(`🎉  Thanks to our ${contributors.length} contributors for this release:`);
+		log.info(contributors.map((contributor) => colors.green(contributor)).join(", "));
+	}
+})();