diff --git a/src/trackers/documentTracker.ts b/src/trackers/documentTracker.ts
index aad0ca6..f6b2aeb 100644
--- a/src/trackers/documentTracker.ts
+++ b/src/trackers/documentTracker.ts
@@ -78,8 +78,8 @@ export class DocumentTracker<T> implements Disposable {
 			workspace.onDidChangeTextDocument(Functions.debounce(this.onTextDocumentChanged, 50), this),
 			workspace.onDidCloseTextDocument(this.onTextDocumentClosed, this),
 			workspace.onDidSaveTextDocument(this.onTextDocumentSaved, this),
-			this.container.git.onDidChangeRepositories(Functions.debounce(this.onRepositoriesChanged, 250), this),
-			this.container.git.onDidChangeRepository(Functions.debounce(this.onRepositoryChanged, 250), this),
+			this.container.git.onDidChangeRepositories(this.onRepositoriesChanged, this),
+			this.container.git.onDidChangeRepository(this.onRepositoryChanged, this),
 		);
 
 		this._dirtyIdleTriggerDelay = configuration.get('advanced.blame.delayAfterEdit');
@@ -144,8 +144,12 @@ export class DocumentTracker<T> implements Disposable {
 		}
 	}
 
-	private onRepositoriesChanged(_e: RepositoriesChangeEvent) {
-		this.reset('repository');
+	private onRepositoriesChanged(e: RepositoriesChangeEvent) {
+		this.reset(
+			'repository',
+			e.added.length ? new Set<string>(e.added.map(r => r.path)) : undefined,
+			e.removed.length ? new Set<string>(e.removed.map(r => r.path)) : undefined,
+		);
 	}
 
 	private onRepositoryChanged(e: RepositoryChangeEvent) {
@@ -158,9 +162,7 @@ export class DocumentTracker<T> implements Disposable {
 				RepositoryChangeComparisonMode.Any,
 			)
 		) {
-			void Promise.allSettled(
-				Iterables.map(this._documentMap.values(), async d => (await d).reset('repository')),
-			);
+			void this.reset('repository', new Set([e.repository.path]));
 		}
 	}
 
@@ -201,14 +203,8 @@ export class DocumentTracker<T> implements Disposable {
 		this.fireDocumentDirtyStateChanged({ editor: editor, document: doc, dirty: doc.dirty });
 	}
 
-	private async onTextDocumentClosed(document: TextDocument) {
-		const doc = this._documentMap.get(document);
-		if (doc == null) return;
-
-		this._documentMap.delete(document);
-		this._documentMap.delete(GitUri.toKey(document.uri));
-
-		(await doc).dispose();
+	private onTextDocumentClosed(document: TextDocument) {
+		void this.remove(document);
 	}
 
 	private async onTextDocumentSaved(document: TextDocument) {
@@ -274,6 +270,18 @@ export class DocumentTracker<T> implements Disposable {
 		return this._documentMap.has(key);
 	}
 
+	private async remove(document: TextDocument, tracked?: TrackedDocument<T>): Promise<void> {
+		let promise;
+		if (tracked != null) {
+			promise = this._documentMap.get(document);
+		}
+
+		this._documentMap.delete(document);
+		this._documentMap.delete(GitUri.toKey(document.uri));
+
+		(tracked ?? (await promise))?.dispose();
+	}
+
 	private async _add(documentOrId: TextDocument | Uri): Promise<TrackedDocument<T>> {
 		let document;
 		if (GitUri.is(documentOrId)) {
@@ -386,11 +394,21 @@ export class DocumentTracker<T> implements Disposable {
 		this._dirtyStateChangedDebounced(e);
 	}
 
-	private reset(reason: 'config' | 'dispose' | 'document' | 'repository') {
+	private reset(reason: 'config' | 'repository', changedRepoPaths?: Set<string>, removedRepoPaths?: Set<string>) {
 		void Promise.allSettled(
 			Iterables.map(
 				Iterables.filter(this._documentMap, ([key]) => typeof key === 'string'),
-				async ([, d]) => (await d).reset(reason),
+				async ([, promise]) => {
+					const doc = await promise;
+					if (removedRepoPaths?.has(doc.uri.repoPath!)) {
+						void this.remove(doc.document, doc);
+						return;
+					}
+
+					if (changedRepoPaths == null || changedRepoPaths.has(doc.uri.repoPath!)) {
+						doc.reset(reason);
+					}
+				},
 			),
 		);
 	}
diff --git a/src/trackers/trackedDocument.ts b/src/trackers/trackedDocument.ts
index f741c9b..1c60564 100644
--- a/src/trackers/trackedDocument.ts
+++ b/src/trackers/trackedDocument.ts
@@ -5,6 +5,7 @@ import { Container } from '../container';
 import { GitUri } from '../git/gitUri';
 import { GitRevision } from '../git/models';
 import { Logger } from '../logger';
+import { Functions } from '../system';
 
 export interface DocumentBlameStateChangeEvent<T> {
 	readonly editor: TextEditor;
@@ -37,7 +38,7 @@ export class TrackedDocument<T> implements Disposable {
 	private _uri!: GitUri;
 
 	private constructor(
-		private readonly _document: TextDocument,
+		readonly document: TextDocument,
 		public readonly key: string,
 		public dirty: boolean,
 		private _eventDelegates: { onDidBlameStateChange(e: DocumentBlameStateChangeEvent<T>): void },
@@ -45,14 +46,15 @@ export class TrackedDocument<T> implements Disposable {
 	) {}
 
 	dispose() {
+		this.state = undefined;
+
 		this._disposed = true;
-		this.reset('dispose');
 		this._disposable?.dispose();
 	}
 
 	private initializing = true;
 	private async initialize(): Promise<void> {
-		const uri = this._document.uri;
+		const uri = this.document.uri;
 
 		this._uri = await GitUri.fromUri(uri);
 		if (!this._disposed) {
@@ -94,10 +96,10 @@ export class TrackedDocument<T> implements Disposable {
 	}
 
 	get lineCount(): number {
-		return this._document.lineCount;
+		return this.document.lineCount;
 	}
 
-	get uri() {
+	get uri(): GitUri {
 		return this._uri;
 	}
 
@@ -109,26 +111,29 @@ export class TrackedDocument<T> implements Disposable {
 	}
 
 	is(document: TextDocument) {
-		return document === this._document;
+		return document === this.document;
 	}
 
-	reset(reason: 'config' | 'dispose' | 'document' | 'repository') {
+	private _updateDebounced:
+		| Functions.Deferrable<({ forceBlameChange }?: { forceBlameChange?: boolean | undefined }) => Promise<void>>
+		| undefined;
+
+	reset(reason: 'config' | 'document' | 'repository') {
 		this._requiresUpdate = true;
 		this._blameFailed = false;
 		this._isDirtyIdle = false;
 
 		if (this.state != null) {
-			// // Don't remove broken blame on change (since otherwise we'll have to run the broken blame again)
-			// if (!this.state.hasErrors) {
-
 			this.state = undefined;
 			Logger.log(`Reset state for '${this.key}', reason=${reason}`);
-
-			// }
 		}
 
-		if (reason === 'repository' && isActiveDocument(this._document)) {
-			void this.update();
+		if (reason === 'repository' && isActiveDocument(this.document)) {
+			if (this._updateDebounced == null) {
+				this._updateDebounced = Functions.debounce(this.update.bind(this), 250);
+			}
+
+			void this._updateDebounced();
 		}
 	}
 
@@ -138,7 +143,7 @@ export class TrackedDocument<T> implements Disposable {
 
 		this._blameFailed = true;
 
-		if (wasBlameable && isActiveDocument(this._document)) {
+		if (wasBlameable && isActiveDocument(this.document)) {
 			void this.update({ forceBlameChange: true });
 		}
 	}
@@ -165,7 +170,7 @@ export class TrackedDocument<T> implements Disposable {
 		this._isDirtyIdle = false;
 
 		// Caches these before the awaits
-		const active = getEditorIfActive(this._document);
+		const active = getEditorIfActive(this.document);
 		const wasBlameable = forceBlameChange ? undefined : this.isBlameable;
 
 		const repo = await this.container.git.getRepository(this._uri);