Adds new log parsers

Uses new log parser for contributors & refs
Removes shortlog references

src/env/node/git/git.ts

@@ -677,8 +677,8 @@ export class Git {
 		ref: string | undefined,
 		{
 			all,
+			argsOrFormat,
 			authors,
-			format = 'default',
 			limit,
 			merges,
 			ordering,
@@ -687,8 +687,8 @@ export class Git {
 			since,
 		}: {
 			all?: boolean;
+			argsOrFormat?: string | string[];
 			authors?: string[];
-			format?: 'default' | 'refs' | 'shortlog' | 'shortlog+stats';
 			limit?: number;
 			merges?: boolean;
 			ordering?: string | null;
@@ -697,26 +697,22 @@ export class Git {
 			since?: string;
 		},
 	) {
+		if (argsOrFormat == null) {
+			argsOrFormat = ['--name-status', `--format=${GitLogParser.defaultFormat}`];
+		}
+
+		if (typeof argsOrFormat === 'string') {
+			argsOrFormat = [`--format=${argsOrFormat}`];
+		}
+
 		const params = [
 			'log',
-			`--format=${
-				format === 'refs'
-					? GitLogParser.simpleRefs
-					: format === 'shortlog' || format === 'shortlog+stats'
-					? GitLogParser.shortlog
-					: GitLogParser.defaultFormat
-			}`,
+			...argsOrFormat,
 			'--full-history',
 			`-M${similarityThreshold == null ? '' : `${similarityThreshold}%`}`,
 			'-m',
 		];
 
-		if (format === 'default') {
-			params.push('--name-status');
-		} else if (format === 'shortlog+stats') {
-			params.push('--shortstat');
-		}
-
 		if (ordering) {
 			params.push(`--${ordering}-order`);
 		}
@@ -734,9 +730,10 @@ export class Git {
 		}
 
 		if (authors != null && authors.length !== 0) {
-			params.push('--use-mailmap', ...authors.map(a => `--author=${a}`));
-		} else if (format === 'shortlog') {
-			params.push('--use-mailmap');
+			if (!params.includes('--use-mailmap')) {
+				params.push('--use-mailmap');
+			}
+			params.push(...authors.map(a => `--author=${a}`));
 		}
 
 		if (all) {
@@ -1272,10 +1269,6 @@ export class Git {
 		return data.length === 0 ? undefined : data.trim();
 	}
 
-	shortlog(repoPath: string) {
-		return this.git<string>({ cwd: repoPath }, 'shortlog', '-sne', '--all', '--no-merges', 'HEAD');
-	}
-
 	async show<TOut extends string | Buffer>(
 		repoPath: string | undefined,
 		fileName: string,

src/env/node/git/localGitProvider.ts

@@ -81,7 +81,6 @@ import {
 	GitLogParser,
 	GitReflogParser,
 	GitRemoteParser,
-	GitShortLogParser,
 	GitStashParser,
 	GitStatusParser,
 	GitTagParser,
@@ -1444,15 +1443,79 @@ export class LocalGitProvider implements GitProvider, Disposable {
 		if (contributors == null) {
 			async function load(this: LocalGitProvider) {
 				try {
+					repoPath = normalizePath(repoPath);
 					const currentUser = await this.getCurrentUser(repoPath);
 
+					const parser = GitLogParser.create<{
+						sha: string;
+						author: string;
+						email: string;
+						date: string;
+						stats?: { files: number; additions: number; deletions: number };
+					}>(
+						{
+							sha: '%H',
+							author: '%aN',
+							email: '%aE',
+							date: '%at',
+						},
+						options?.stats
+							? {
+									additionalArgs: ['--shortstat', '--use-mailmap'],
+									parseEntry: (fields, entry) => {
+										const line = fields.next().value;
+										const match = GitLogParser.shortstatRegex.exec(line);
+										if (match?.groups != null) {
+											const { files, additions, deletions } = match.groups;
+											entry.stats = {
+												files: Number(files || 0),
+												additions: Number(additions || 0),
+												deletions: Number(deletions || 0),
+											};
+										}
+										return entry;
+									},
+									prefix: '%x00',
+									fieldSuffix: '%x00',
+									skip: 1,
+							  }
+							: undefined,
+					);
+
 					const data = await this.git.log(repoPath, options?.ref, {
 						all: options?.all,
-						format: options?.stats ? 'shortlog+stats' : 'shortlog',
+						argsOrFormat: parser.arguments,
 					});
-					const shortlog = GitShortLogParser.parseFromLog(data, repoPath, currentUser);
 
-					return shortlog != null ? shortlog.contributors : [];
+					const contributors = new Map<string, GitContributor>();
+
+					const commits = parser.parse(data);
+					for (const c of commits) {
+						const key = `${c.author}|${c.email}`;
+						let contributor = contributors.get(key);
+						if (contributor == null) {
+							contributor = new GitContributor(
+								repoPath,
+								c.author,
+								c.email,
+								1,
+								new Date(Number(c.date) * 1000),
+								c.stats,
+								currentUser != null
+									? currentUser.name === c.author && currentUser.email === c.email
+									: false,
+							);
+							contributors.set(key, contributor);
+						} else {
+							(contributor as PickMutable<GitContributor, 'count'>).count++;
+							const date = new Date(Number(c.date) * 1000);
+							if (date > contributor.date) {
+								(contributor as PickMutable<GitContributor, 'date'>).date = date;
+							}
+						}
+					}
+
+					return [...contributors.values()];
 				} catch (ex) {
 					this._contributorsCache.delete(key);
 
@@ -1895,9 +1958,11 @@ export class LocalGitProvider implements GitProvider, Disposable {
 		const limit = options?.limit ?? this.container.config.advanced.maxListItems ?? 0;
 
 		try {
+			const parser = GitLogParser.createSingle('%H');
+
 			const data = await this.git.log(repoPath, options?.ref, {
 				authors: options?.authors,
-				format: 'refs',
+				argsOrFormat: parser.arguments,
 				limit: limit,
 				merges: options?.merges == null ? true : options.merges,
 				reverse: options?.reverse,
@@ -1905,8 +1970,9 @@ export class LocalGitProvider implements GitProvider, Disposable {
 				since: options?.since,
 				ordering: options?.ordering ?? this.container.config.advanced.commitOrdering,
 			});
-			const commits = GitLogParser.parseRefsOnly(data);
-			return new Set(commits);
+
+			const commits = new Set(parser.parse(data));
+			return commits;
 		} catch (ex) {
 			Logger.error(ex, cc);
 			debugger;

src/git/parsers.ts

@@ -4,7 +4,6 @@ export * from './parsers/diffParser';
 export * from './parsers/logParser';
 export * from './parsers/reflogParser';
 export * from './parsers/remoteParser';
-export * from './parsers/shortlogParser';
 export * from './parsers/stashParser';
 export * from './parsers/statusParser';
 export * from './parsers/tagParser';

src/git/parsers/logParser.ts

@@ -62,7 +62,50 @@ interface LogEntry {
 	line?: GitLogCommitLine;
 }
 
+export type Parser<T> = {
+	arguments: string[];
+	parse: (data: string) => Generator<T>;
+};
+
+type ParsedEntryFile = { status: string; path: string; originalPath?: string };
+type ParsedEntryWithFiles<T> = { [K in keyof T]: string } & { files: ParsedEntryFile[] };
+type ParserWithFiles<T> = {
+	arguments: string[];
+	parse: (data: string) => Generator<ParsedEntryWithFiles<T>>;
+};
+
 export class GitLogParser {
+	static readonly shortstatRegex =
+		/(?<files>\d+) files? changed(?:, (?<additions>\d+) insertions?\(\+\))?(?:, (?<deletions>\d+) deletions?\(-\))?/;
+
+	private static _defaultParser: ParserWithFiles<{
+		sha: string;
+		author: string;
+		authorEmail: string;
+		authorDate: string;
+		committer: string;
+		committerEmail: string;
+		committerDate: string;
+		message: string;
+		parents: string[];
+	}>;
+	static get defaultParser() {
+		if (this._defaultParser == null) {
+			this._defaultParser = GitLogParser.createWithFiles({
+				sha: '%H',
+				author: '%aN',
+				authorEmail: '%aE',
+				authorDate: '%at',
+				committer: '%cN',
+				committerEmail: '%cE',
+				committerDate: '%ct',
+				message: '%B',
+				parents: '%P',
+			});
+		}
+		return this._defaultParser;
+	}
+
 	static defaultFormat = [
 		`${lb}${sl}f${rb}`,
 		`${lb}r${rb}${sp}%H`, // ref
@@ -82,6 +125,141 @@ export class GitLogParser {
 
 	static shortlog = '%H%x00%aN%x00%aE%x00%at';
 
+	static create<T extends Record<string, unknown>>(
+		fieldMapping: ExtractAll<T, string>,
+		options?: {
+			additionalArgs?: string[];
+			parseEntry?: (fields: IterableIterator<string>, entry: T) => void;
+			prefix?: string;
+			fieldPrefix?: string;
+			fieldSuffix?: string;
+			separator?: string;
+			skip?: number;
+		},
+	): Parser<T> {
+		let format = options?.prefix ?? '';
+		const keys: (keyof ExtractAll<T, string>)[] = [];
+		for (const key in fieldMapping) {
+			keys.push(key);
+			format += `${options?.fieldPrefix ?? ''}${fieldMapping[key]}${
+				options?.fieldSuffix ?? (options?.fieldPrefix == null ? '%x00' : '')
+			}`;
+		}
+
+		const args = ['-z', `--format=${format}`];
+		if (options?.additionalArgs != null && options.additionalArgs.length > 0) {
+			args.push(...options.additionalArgs);
+		}
+
+		function* parse(data: string): Generator<T> {
+			let entry: T = {} as any;
+			let fieldCount = 0;
+			let field;
+
+			const fields = getLines(data, options?.separator ?? '\0');
+			if (options?.skip) {
+				for (let i = 0; i < options.skip; i++) {
+					field = fields.next();
+				}
+			}
+
+			while (true) {
+				field = fields.next();
+				if (field.done) break;
+
+				entry[keys[fieldCount++]] = field.value as T[keyof T];
+
+				if (fieldCount === keys.length) {
+					fieldCount = 0;
+					field = fields.next();
+
+					options?.parseEntry?.(fields, entry);
+					yield entry;
+
+					entry = {} as any;
+				}
+			}
+		}
+
+		return { arguments: args, parse: parse };
+	}
+
+	static createSingle(field: string): Parser<string> {
+		const format = field;
+		const args = ['-z', `--format=${format}`];
+
+		function* parse(data: string): Generator<string> {
+			let field;
+
+			const fields = getLines(data, '\0');
+			while (true) {
+				field = fields.next();
+				if (field.done) break;
+
+				yield field.value;
+			}
+		}
+
+		return { arguments: args, parse: parse };
+	}
+
+	static createWithFiles<T extends Record<string, string>>(fieldMapping: T): ParserWithFiles<T> {
+		let format = '%x00%x00';
+		const keys: (keyof T)[] = [];
+		for (const key in fieldMapping) {
+			keys.push(key);
+			format += `%x00${fieldMapping[key]}`;
+		}
+
+		const args = ['-z', `--format=${format}`, '--name-status'];
+
+		function* parse(data: string): Generator<ParsedEntryWithFiles<T>> {
+			const records = getLines(data, '\0\0\0\0');
+
+			let entry: ParsedEntryWithFiles<T>;
+			let files: ParsedEntryFile[];
+			let fields: IterableIterator<string>;
+
+			let first = true;
+			for (let record of records) {
+				if (first) {
+					first = false;
+					// Fix the first record (since it only has 3 nulls)
+					record = record.slice(3);
+				}
+
+				entry = {} as any;
+				files = [];
+				fields = getLines(record, '\0');
+
+				let fieldCount = 0;
+				let field;
+				while (true) {
+					field = fields.next();
+					if (field.done) break;
+
+					if (fieldCount < keys.length) {
+						entry[keys[fieldCount++]] = field.value as ParsedEntryWithFiles<T>[keyof T];
+					} else {
+						const file: ParsedEntryFile = { status: field.value.trim(), path: undefined! };
+						field = fields.next();
+						file.path = field.value;
+
+						if (file.status[0] === 'R' || file.status[0] === 'C') {
+							field = fields.next();
+							file.originalPath = field.value;
+						}
+					}
+				}
+
+				entry.files = files;
+				yield entry;
+			}
+		}
+
+		return { arguments: args, parse: parse };
+	}
+
 	@debug({ args: false })
 	static parse(
 		data: string,

src/git/parsers/shortlogParser.ts

@@ -1,137 +0,0 @@
-import { debug } from '../../system';
-import { GitContributor, GitShortLog, GitUser } from '../models';
-
-const shortlogRegex = /^(.*?)\t(.*?) <(.*?)>$/gm;
-const shortstatRegex =
-	/(?<files>\d+) files? changed(?:, (?<additions>\d+) insertions?\(\+\))?(?:, (?<deletions>\d+) deletions?\(-\))?/;
-
-export class GitShortLogParser {
-	@debug({ args: false, singleLine: true })
-	static parse(data: string, repoPath: string): GitShortLog | undefined {
-		if (!data) return undefined;
-
-		const contributors: GitContributor[] = [];
-
-		let count;
-		let name;
-		let email;
-
-		let match;
-		do {
-			match = shortlogRegex.exec(data);
-			if (match == null) break;
-
-			[, count, name, email] = match;
-
-			contributors.push(
-				new GitContributor(
-					repoPath,
-					// Stops excessive memory usage -- https://bugs.chromium.org/p/v8/issues/detail?id=2869
-					` ${name}`.substr(1),
-					// Stops excessive memory usage -- https://bugs.chromium.org/p/v8/issues/detail?id=2869
-					` ${email}`.substr(1),
-					Number(count) || 0,
-					new Date(),
-				),
-			);
-		} while (true);
-
-		return { repoPath: repoPath, contributors: contributors };
-	}
-
-	@debug({ args: false })
-	static parseFromLog(data: string, repoPath: string, currentUser?: GitUser): GitShortLog | undefined {
-		if (!data) return undefined;
-
-		type Contributor = {
-			sha: string;
-			name: string;
-			email: string;
-			count: number;
-			timestamp: number;
-			stats?: {
-				files: number;
-				additions: number;
-				deletions: number;
-			};
-		};
-
-		const contributors = new Map<string, Contributor>();
-
-		const lines = data.trim().split('\n');
-		for (let i = 0; i < lines.length; i++) {
-			const [sha, author, email, date] = lines[i].trim().split('\0');
-
-			let stats:
-				| {
-						files: number;
-						additions: number;
-						deletions: number;
-				  }
-				| undefined;
-			if (lines[i + 1] === '') {
-				i += 2;
-				const match = shortstatRegex.exec(lines[i]);
-
-				if (match?.groups != null) {
-					const { files, additions, deletions } = match.groups;
-					stats = {
-						files: Number(files || 0),
-						additions: Number(additions || 0),
-						deletions: Number(deletions || 0),
-					};
-				}
-			}
-
-			const timestamp = Number(date);
-
-			const contributor = contributors.get(`${author}${email}`);
-			if (contributor == null) {
-				contributors.set(`${author}${email}`, {
-					sha: sha,
-					name: author,
-					email: email,
-					count: 1,
-					timestamp: timestamp,
-					stats: stats,
-				});
-			} else {
-				contributor.count++;
-				if (stats != null) {
-					if (contributor.stats == null) {
-						contributor.stats = stats;
-					} else {
-						contributor.stats.files += stats.files;
-						contributor.stats.additions += stats.additions;
-						contributor.stats.deletions += stats.deletions;
-					}
-				}
-				if (timestamp > contributor.timestamp) {
-					contributor.timestamp = timestamp;
-				}
-			}
-		}
-
-		return {
-			repoPath: repoPath,
-			contributors:
-				contributors.size === 0
-					? []
-					: Array.from(
-							contributors.values(),
-							c =>
-								new GitContributor(
-									repoPath,
-									c.name,
-									c.email,
-									c.count,
-									new Date(Number(c.timestamp) * 1000),
-									c.stats,
-									currentUser != null
-										? currentUser.name === c.name && currentUser.email === c.email
-										: false,
-								),
-					  ),
-		};
-	}
-}

src/system.ts

@@ -6,7 +6,10 @@ declare global {
 	export type PickMutable<T, K extends keyof T> = Omit<T, K> & { -readonly [P in K]: T[P] };
 
 	export type ExcludeSome<T, K extends keyof T, R> = Omit<T, K> & { [P in K]-?: Exclude<T[P], R> };
+
+	export type ExtractAll<T, U> = { [K in keyof T]: T[K] extends U ? T[K] : never };
 	export type ExtractSome<T, K extends keyof T, R> = Omit<T, K> & { [P in K]-?: Extract<T[P], R> };
+
 	export type RequireSome<T, K extends keyof T> = Omit<T, K> & { [P in K]-?: T[P] };
 
 	export type AllNonNullable<T> = { [P in keyof T]-?: NonNullable<T[P]> };

src/vsls/host.ts

@@ -35,7 +35,6 @@ const gitWhitelist = new Map boolean>([
 	['remote', args => args[1] === '-v' || args[1] === 'get-url'],
 	['rev-list', defaultWhitelistFn],
 	['rev-parse', defaultWhitelistFn],
-	['shortlog', defaultWhitelistFn],
 	['show', defaultWhitelistFn],
 	['show-ref', defaultWhitelistFn],
 	['stash', args => args[1] === 'list'],