From 9809dc43f392932845321b36ce518117794e1c00 Mon Sep 17 00:00:00 2001 From: Eric Amodio Date: Mon, 31 Jan 2022 01:44:00 -0500 Subject: [PATCH] Adds new log parsers Uses new log parser for contributors & refs Removes shortlog references --- src/env/node/git/git.ts | 37 +++----- src/env/node/git/localGitProvider.ts | 80 ++++++++++++++-- src/git/parsers.ts | 1 - src/git/parsers/logParser.ts | 178 +++++++++++++++++++++++++++++++++++ src/git/parsers/shortlogParser.ts | 137 --------------------------- src/system.ts | 3 + src/vsls/host.ts | 1 - 7 files changed, 269 insertions(+), 168 deletions(-) delete mode 100644 src/git/parsers/shortlogParser.ts diff --git a/src/env/node/git/git.ts b/src/env/node/git/git.ts index cc36f6b..bb666ce 100644 --- a/src/env/node/git/git.ts +++ b/src/env/node/git/git.ts @@ -677,8 +677,8 @@ export class Git { ref: string | undefined, { all, + argsOrFormat, authors, - format = 'default', limit, merges, ordering, @@ -687,8 +687,8 @@ export class Git { since, }: { all?: boolean; + argsOrFormat?: string | string[]; authors?: string[]; - format?: 'default' | 'refs' | 'shortlog' | 'shortlog+stats'; limit?: number; merges?: boolean; ordering?: string | null; @@ -697,26 +697,22 @@ export class Git { since?: string; }, ) { + if (argsOrFormat == null) { + argsOrFormat = ['--name-status', `--format=${GitLogParser.defaultFormat}`]; + } + + if (typeof argsOrFormat === 'string') { + argsOrFormat = [`--format=${argsOrFormat}`]; + } + const params = [ 'log', - `--format=${ - format === 'refs' - ? GitLogParser.simpleRefs - : format === 'shortlog' || format === 'shortlog+stats' - ? GitLogParser.shortlog - : GitLogParser.defaultFormat - }`, + ...argsOrFormat, '--full-history', `-M${similarityThreshold == null ? '' : `${similarityThreshold}%`}`, '-m', ]; - if (format === 'default') { - params.push('--name-status'); - } else if (format === 'shortlog+stats') { - params.push('--shortstat'); - } - if (ordering) { params.push(`--${ordering}-order`); } @@ -734,9 +730,10 @@ export class Git { } if (authors != null && authors.length !== 0) { - params.push('--use-mailmap', ...authors.map(a => `--author=${a}`)); - } else if (format === 'shortlog') { - params.push('--use-mailmap'); + if (!params.includes('--use-mailmap')) { + params.push('--use-mailmap'); + } + params.push(...authors.map(a => `--author=${a}`)); } if (all) { @@ -1272,10 +1269,6 @@ export class Git { return data.length === 0 ? undefined : data.trim(); } - shortlog(repoPath: string) { - return this.git({ cwd: repoPath }, 'shortlog', '-sne', '--all', '--no-merges', 'HEAD'); - } - async show( repoPath: string | undefined, fileName: string, diff --git a/src/env/node/git/localGitProvider.ts b/src/env/node/git/localGitProvider.ts index 21541a6..30f93a8 100644 --- a/src/env/node/git/localGitProvider.ts +++ b/src/env/node/git/localGitProvider.ts @@ -81,7 +81,6 @@ import { GitLogParser, GitReflogParser, GitRemoteParser, - GitShortLogParser, GitStashParser, GitStatusParser, GitTagParser, @@ -1444,15 +1443,79 @@ export class LocalGitProvider implements GitProvider, Disposable { if (contributors == null) { async function load(this: LocalGitProvider) { try { + repoPath = normalizePath(repoPath); const currentUser = await this.getCurrentUser(repoPath); + const parser = GitLogParser.create<{ + sha: string; + author: string; + email: string; + date: string; + stats?: { files: number; additions: number; deletions: number }; + }>( + { + sha: '%H', + author: '%aN', + email: '%aE', + date: '%at', + }, + options?.stats + ? { + additionalArgs: ['--shortstat', '--use-mailmap'], + parseEntry: (fields, entry) => { + const line = fields.next().value; + const match = GitLogParser.shortstatRegex.exec(line); + if (match?.groups != null) { + const { files, additions, deletions } = match.groups; + entry.stats = { + files: Number(files || 0), + additions: Number(additions || 0), + deletions: Number(deletions || 0), + }; + } + return entry; + }, + prefix: '%x00', + fieldSuffix: '%x00', + skip: 1, + } + : undefined, + ); + const data = await this.git.log(repoPath, options?.ref, { all: options?.all, - format: options?.stats ? 'shortlog+stats' : 'shortlog', + argsOrFormat: parser.arguments, }); - const shortlog = GitShortLogParser.parseFromLog(data, repoPath, currentUser); - return shortlog != null ? shortlog.contributors : []; + const contributors = new Map(); + + const commits = parser.parse(data); + for (const c of commits) { + const key = `${c.author}|${c.email}`; + let contributor = contributors.get(key); + if (contributor == null) { + contributor = new GitContributor( + repoPath, + c.author, + c.email, + 1, + new Date(Number(c.date) * 1000), + c.stats, + currentUser != null + ? currentUser.name === c.author && currentUser.email === c.email + : false, + ); + contributors.set(key, contributor); + } else { + (contributor as PickMutable).count++; + const date = new Date(Number(c.date) * 1000); + if (date > contributor.date) { + (contributor as PickMutable).date = date; + } + } + } + + return [...contributors.values()]; } catch (ex) { this._contributorsCache.delete(key); @@ -1895,9 +1958,11 @@ export class LocalGitProvider implements GitProvider, Disposable { const limit = options?.limit ?? this.container.config.advanced.maxListItems ?? 0; try { + const parser = GitLogParser.createSingle('%H'); + const data = await this.git.log(repoPath, options?.ref, { authors: options?.authors, - format: 'refs', + argsOrFormat: parser.arguments, limit: limit, merges: options?.merges == null ? true : options.merges, reverse: options?.reverse, @@ -1905,8 +1970,9 @@ export class LocalGitProvider implements GitProvider, Disposable { since: options?.since, ordering: options?.ordering ?? this.container.config.advanced.commitOrdering, }); - const commits = GitLogParser.parseRefsOnly(data); - return new Set(commits); + + const commits = new Set(parser.parse(data)); + return commits; } catch (ex) { Logger.error(ex, cc); debugger; diff --git a/src/git/parsers.ts b/src/git/parsers.ts index aee87af..5e2bbba 100644 --- a/src/git/parsers.ts +++ b/src/git/parsers.ts @@ -4,7 +4,6 @@ export * from './parsers/diffParser'; export * from './parsers/logParser'; export * from './parsers/reflogParser'; export * from './parsers/remoteParser'; -export * from './parsers/shortlogParser'; export * from './parsers/stashParser'; export * from './parsers/statusParser'; export * from './parsers/tagParser'; diff --git a/src/git/parsers/logParser.ts b/src/git/parsers/logParser.ts index 765da34..a62af01 100644 --- a/src/git/parsers/logParser.ts +++ b/src/git/parsers/logParser.ts @@ -62,7 +62,50 @@ interface LogEntry { line?: GitLogCommitLine; } +export type Parser = { + arguments: string[]; + parse: (data: string) => Generator; +}; + +type ParsedEntryFile = { status: string; path: string; originalPath?: string }; +type ParsedEntryWithFiles = { [K in keyof T]: string } & { files: ParsedEntryFile[] }; +type ParserWithFiles = { + arguments: string[]; + parse: (data: string) => Generator>; +}; + export class GitLogParser { + static readonly shortstatRegex = + /(?\d+) files? changed(?:, (?\d+) insertions?\(\+\))?(?:, (?\d+) deletions?\(-\))?/; + + private static _defaultParser: ParserWithFiles<{ + sha: string; + author: string; + authorEmail: string; + authorDate: string; + committer: string; + committerEmail: string; + committerDate: string; + message: string; + parents: string[]; + }>; + static get defaultParser() { + if (this._defaultParser == null) { + this._defaultParser = GitLogParser.createWithFiles({ + sha: '%H', + author: '%aN', + authorEmail: '%aE', + authorDate: '%at', + committer: '%cN', + committerEmail: '%cE', + committerDate: '%ct', + message: '%B', + parents: '%P', + }); + } + return this._defaultParser; + } + static defaultFormat = [ `${lb}${sl}f${rb}`, `${lb}r${rb}${sp}%H`, // ref @@ -82,6 +125,141 @@ export class GitLogParser { static shortlog = '%H%x00%aN%x00%aE%x00%at'; + static create>( + fieldMapping: ExtractAll, + options?: { + additionalArgs?: string[]; + parseEntry?: (fields: IterableIterator, entry: T) => void; + prefix?: string; + fieldPrefix?: string; + fieldSuffix?: string; + separator?: string; + skip?: number; + }, + ): Parser { + let format = options?.prefix ?? ''; + const keys: (keyof ExtractAll)[] = []; + for (const key in fieldMapping) { + keys.push(key); + format += `${options?.fieldPrefix ?? ''}${fieldMapping[key]}${ + options?.fieldSuffix ?? (options?.fieldPrefix == null ? '%x00' : '') + }`; + } + + const args = ['-z', `--format=${format}`]; + if (options?.additionalArgs != null && options.additionalArgs.length > 0) { + args.push(...options.additionalArgs); + } + + function* parse(data: string): Generator { + let entry: T = {} as any; + let fieldCount = 0; + let field; + + const fields = getLines(data, options?.separator ?? '\0'); + if (options?.skip) { + for (let i = 0; i < options.skip; i++) { + field = fields.next(); + } + } + + while (true) { + field = fields.next(); + if (field.done) break; + + entry[keys[fieldCount++]] = field.value as T[keyof T]; + + if (fieldCount === keys.length) { + fieldCount = 0; + field = fields.next(); + + options?.parseEntry?.(fields, entry); + yield entry; + + entry = {} as any; + } + } + } + + return { arguments: args, parse: parse }; + } + + static createSingle(field: string): Parser { + const format = field; + const args = ['-z', `--format=${format}`]; + + function* parse(data: string): Generator { + let field; + + const fields = getLines(data, '\0'); + while (true) { + field = fields.next(); + if (field.done) break; + + yield field.value; + } + } + + return { arguments: args, parse: parse }; + } + + static createWithFiles>(fieldMapping: T): ParserWithFiles { + let format = '%x00%x00'; + const keys: (keyof T)[] = []; + for (const key in fieldMapping) { + keys.push(key); + format += `%x00${fieldMapping[key]}`; + } + + const args = ['-z', `--format=${format}`, '--name-status']; + + function* parse(data: string): Generator> { + const records = getLines(data, '\0\0\0\0'); + + let entry: ParsedEntryWithFiles; + let files: ParsedEntryFile[]; + let fields: IterableIterator; + + let first = true; + for (let record of records) { + if (first) { + first = false; + // Fix the first record (since it only has 3 nulls) + record = record.slice(3); + } + + entry = {} as any; + files = []; + fields = getLines(record, '\0'); + + let fieldCount = 0; + let field; + while (true) { + field = fields.next(); + if (field.done) break; + + if (fieldCount < keys.length) { + entry[keys[fieldCount++]] = field.value as ParsedEntryWithFiles[keyof T]; + } else { + const file: ParsedEntryFile = { status: field.value.trim(), path: undefined! }; + field = fields.next(); + file.path = field.value; + + if (file.status[0] === 'R' || file.status[0] === 'C') { + field = fields.next(); + file.originalPath = field.value; + } + } + } + + entry.files = files; + yield entry; + } + } + + return { arguments: args, parse: parse }; + } + @debug({ args: false }) static parse( data: string, diff --git a/src/git/parsers/shortlogParser.ts b/src/git/parsers/shortlogParser.ts deleted file mode 100644 index 3153cf7..0000000 --- a/src/git/parsers/shortlogParser.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { debug } from '../../system'; -import { GitContributor, GitShortLog, GitUser } from '../models'; - -const shortlogRegex = /^(.*?)\t(.*?) <(.*?)>$/gm; -const shortstatRegex = - /(?\d+) files? changed(?:, (?\d+) insertions?\(\+\))?(?:, (?\d+) deletions?\(-\))?/; - -export class GitShortLogParser { - @debug({ args: false, singleLine: true }) - static parse(data: string, repoPath: string): GitShortLog | undefined { - if (!data) return undefined; - - const contributors: GitContributor[] = []; - - let count; - let name; - let email; - - let match; - do { - match = shortlogRegex.exec(data); - if (match == null) break; - - [, count, name, email] = match; - - contributors.push( - new GitContributor( - repoPath, - // Stops excessive memory usage -- https://bugs.chromium.org/p/v8/issues/detail?id=2869 - ` ${name}`.substr(1), - // Stops excessive memory usage -- https://bugs.chromium.org/p/v8/issues/detail?id=2869 - ` ${email}`.substr(1), - Number(count) || 0, - new Date(), - ), - ); - } while (true); - - return { repoPath: repoPath, contributors: contributors }; - } - - @debug({ args: false }) - static parseFromLog(data: string, repoPath: string, currentUser?: GitUser): GitShortLog | undefined { - if (!data) return undefined; - - type Contributor = { - sha: string; - name: string; - email: string; - count: number; - timestamp: number; - stats?: { - files: number; - additions: number; - deletions: number; - }; - }; - - const contributors = new Map(); - - const lines = data.trim().split('\n'); - for (let i = 0; i < lines.length; i++) { - const [sha, author, email, date] = lines[i].trim().split('\0'); - - let stats: - | { - files: number; - additions: number; - deletions: number; - } - | undefined; - if (lines[i + 1] === '') { - i += 2; - const match = shortstatRegex.exec(lines[i]); - - if (match?.groups != null) { - const { files, additions, deletions } = match.groups; - stats = { - files: Number(files || 0), - additions: Number(additions || 0), - deletions: Number(deletions || 0), - }; - } - } - - const timestamp = Number(date); - - const contributor = contributors.get(`${author}${email}`); - if (contributor == null) { - contributors.set(`${author}${email}`, { - sha: sha, - name: author, - email: email, - count: 1, - timestamp: timestamp, - stats: stats, - }); - } else { - contributor.count++; - if (stats != null) { - if (contributor.stats == null) { - contributor.stats = stats; - } else { - contributor.stats.files += stats.files; - contributor.stats.additions += stats.additions; - contributor.stats.deletions += stats.deletions; - } - } - if (timestamp > contributor.timestamp) { - contributor.timestamp = timestamp; - } - } - } - - return { - repoPath: repoPath, - contributors: - contributors.size === 0 - ? [] - : Array.from( - contributors.values(), - c => - new GitContributor( - repoPath, - c.name, - c.email, - c.count, - new Date(Number(c.timestamp) * 1000), - c.stats, - currentUser != null - ? currentUser.name === c.name && currentUser.email === c.email - : false, - ), - ), - }; - } -} diff --git a/src/system.ts b/src/system.ts index 1959030..8854c91 100644 --- a/src/system.ts +++ b/src/system.ts @@ -6,7 +6,10 @@ declare global { export type PickMutable = Omit & { -readonly [P in K]: T[P] }; export type ExcludeSome = Omit & { [P in K]-?: Exclude }; + + export type ExtractAll = { [K in keyof T]: T[K] extends U ? T[K] : never }; export type ExtractSome = Omit & { [P in K]-?: Extract }; + export type RequireSome = Omit & { [P in K]-?: T[P] }; export type AllNonNullable = { [P in keyof T]-?: NonNullable }; diff --git a/src/vsls/host.ts b/src/vsls/host.ts index fb206f3..574092f 100644 --- a/src/vsls/host.ts +++ b/src/vsls/host.ts @@ -35,7 +35,6 @@ const gitWhitelist = new Map boolean>([ ['remote', args => args[1] === '-v' || args[1] === 'get-url'], ['rev-list', defaultWhitelistFn], ['rev-parse', defaultWhitelistFn], - ['shortlog', defaultWhitelistFn], ['show', defaultWhitelistFn], ['show-ref', defaultWhitelistFn], ['stash', args => args[1] === 'list'],