External Context Integration Guide

+-----------------------------------------------------------------+
|                    External Systems                              |
+---------------+---------------------+---------------------------+
|    Git        |   Issue Trackers    |        Sentry             |
|  (commits)    |  (Jira/GitHub/GL)   |       (errors)            |
+-------+-------+----------+----------+-----------+---------------+
        |                  |                      |
        v                  v                      v
+-----------------------------------------------------------------+
|              ExternalContextOrchestrator                         |
|  +---------------+ +-----------------+ +---------------------+  |
|  |GitSyncService | |IssueSyncService | |SentrySyncService    |  |
|  +---------------+ +-----------------+ +---------------------+  |
+-----------------------------------------------------------------+
        |                                          |
        v                                          v
+--------------------+                  +------------------------+
|   PostgreSQL       |                  |      DuckDB CPG        |
| (raw metadata)     |                  |   (nodes_tag_v2,       |
| - external_context |                  |    edges_tagged_by,    |
| - file_commit_hist |                  |    nodes_method)        |
| - git_authors      |                  +------------------------+
| - runtime_metrics  |
+--------------------+

pip install -r requirements.txt

# Git (no additional config needed - uses local git)

# GitHub Issues
export GITHUB_TOKEN="ghp_xxxxxxxxxxxx"

# GitLab Issues
export GITLAB_TOKEN="glpat-xxxxxxxxxxxx"

# Jira
export JIRA_TOKEN="your_jira_api_token"

# Sentry
export SENTRY_AUTH_TOKEN="your_sentry_token"

from src.services.external_context import ExternalContextOrchestrator

orchestrator = ExternalContextOrchestrator(
    duckdb_conn=your_duckdb_connection,
    pg_conn=your_postgres_connection,  # optional
    repo_path="/path/to/your/repo"
)

result = orchestrator.sync_all(
    git_config={"since_days": 90, "use_blame": True},
    issue_config={
        "provider": "github",
        "repo": "owner/repo",
        "token": os.getenv("GITHUB_TOKEN"),
    },
    sentry_config={
        "org_slug": "my-org",
        "project_slug": "my-project",
        "token": os.getenv("SENTRY_AUTH_TOKEN"),
    },
)

print(f"Total synced: {result.total_items_synced}, tags: {result.total_tags_created}")

from src.services.external_context import GitSyncService

service = GitSyncService(
    duckdb_conn=conn,
    pg_conn=pg_conn,        # optional, for storing raw data
    repo_path="/path/to/repo",
    max_commits=1000,        # default
    since_days=90            # default
)

# Sync with blame analysis
result = service.sync(use_blame=True)
print(f"Synced {result.items_synced} commits, created {result.tags_created} tags")

# Sync only specific files, without blame
result = service.sync(files=["src/main.py", "src/auth.py"], use_blame=False)

-- Find all methods modified by a specific author
SELECT m.full_name, m.filename, t.value AS author
FROM nodes_method m
JOIN edges_tagged_by e ON m.id = e.src
JOIN nodes_tag_v2 t ON e.dst = t.id
WHERE t.name = 'git-author' AND t.value = 'developer@example.com';

-- Find high-churn code (methods changed > 10 times)
SELECT m.full_name, CAST(t.value AS INT) AS churn_count
FROM nodes_method m
JOIN edges_tagged_by e ON m.id = e.src
JOIN nodes_tag_v2 t ON e.dst = t.id
WHERE t.name = 'git-churn' AND CAST(t.value AS INT) > 10
ORDER BY churn_count DESC;

-- Find bus factor candidates (methods with only 1 author)
SELECT m.full_name, t_author.value AS sole_author
FROM nodes_method m
JOIN edges_tagged_by e1 ON m.id = e1.src
JOIN nodes_tag_v2 t_blame ON e1.dst = t_blame.id
JOIN edges_tagged_by e2 ON m.id = e2.src
JOIN nodes_tag_v2 t_author ON e2.dst = t_author.id
WHERE t_blame.name = 'git-blame-count' AND t_blame.value = '1'
  AND t_author.name = 'git-author';

-- Find recently modified methods (last 7 days)
SELECT m.full_name, m.filename, t.value AS last_modified
FROM nodes_method m
JOIN edges_tagged_by e ON m.id = e.src
JOIN nodes_tag_v2 t ON e.dst = t.id
WHERE t.name = 'git-last-modified'
  AND CAST(t.value AS TIMESTAMP) > NOW() - INTERVAL '7 days'
ORDER BY t.value DESC;

from src.services.external_context import IssueSyncService

# GitHub
service = IssueSyncService(
    duckdb_conn=conn,
    pg_conn=pg_conn,
    provider="github",
    repo="owner/repo",
    token=os.getenv("GITHUB_TOKEN"),
    max_issues=500  # default
)

# GitLab
service = IssueSyncService(
    duckdb_conn=conn,
    provider="gitlab",
    repo="owner/repo",
    base_url="https://gitlab.company.com",
    token=os.getenv("GITLAB_TOKEN")
)

# Jira
service = IssueSyncService(
    duckdb_conn=conn,
    provider="jira",
    project_key="PROJ",
    base_url="https://company.atlassian.net",
    token=os.getenv("JIRA_TOKEN")
)

result = service.sync(since_days=90, link_via_commits=True)

service = IssueSyncService(
    duckdb_conn=conn,
    provider="jira",
    project_key="PROJ",
    base_url="https://company.atlassian.net",
    token=os.getenv("JIRA_TOKEN")
)

issue_key = service.create_jira_issue(
    summary="Fix NullPointerException in AuthService",
    description="Method authenticate() throws NPE when token is expired",
    issue_type="Bug",        # default
    priority="Medium",       # default
    labels=["codegraph", "auto-detected"],
    components=["auth"]
)
# Returns "PROJ-456" or None on failure

from src.services.external_context import SentrySyncService

service = SentrySyncService(
    duckdb_conn=conn,
    pg_conn=pg_conn,
    org_slug="my-organization",
    project_slug="my-project",
    token=os.getenv("SENTRY_AUTH_TOKEN"),
    base_url="https://sentry.io",  # default; override for self-hosted
    max_issues=200                  # default
)

result = service.sync(since_days=30, min_events=1)

-- Find methods with critical error frequency
SELECT m.full_name, m.filename, t.value AS frequency
FROM nodes_method m
JOIN edges_tagged_by e ON m.id = e.src
JOIN nodes_tag_v2 t ON e.dst = t.id
WHERE t.name = 'error-frequency' AND t.value = 'critical';

-- Find methods by error type
SELECT m.full_name, t.value AS error_type
FROM nodes_method m
JOIN edges_tagged_by e ON m.id = e.src
JOIN nodes_tag_v2 t ON e.dst = t.id
WHERE t.name = 'error-type' AND t.value LIKE '%NullPointer%';

-- Combine: methods with high+ frequency AND fatal level
SELECT DISTINCT m.full_name, m.filename
FROM nodes_method m
JOIN edges_tagged_by e1 ON m.id = e1.src
JOIN nodes_tag_v2 t_freq ON e1.dst = t_freq.id
JOIN edges_tagged_by e2 ON m.id = e2.src
JOIN nodes_tag_v2 t_level ON e2.dst = t_level.id
WHERE t_freq.name = 'error-frequency' AND t_freq.value IN ('critical', 'high')
  AND t_level.name = 'error-level' AND t_level.value = 'fatal';

from src.services.external_context import ExternalContextOrchestrator

orchestrator = ExternalContextOrchestrator(
    duckdb_conn=conn,
    pg_conn=pg_conn,       # optional
    repo_path=".",
    vector_store=None      # optional, for semantic indexing
)

# Sync all sources at once
result = orchestrator.sync_all(
    git_config={"since_days": 30, "use_blame": True},
    issue_config={
        "provider": "github",
        "repo": "owner/repo",
        "token": token,
    },
    sentry_config={
        "org_slug": "org",
        "project_slug": "proj",
        "token": sentry_token,
    },
    parallel=True  # run syncs in parallel
)

print(f"Success: {result.success}")
print(f"Items synced: {result.total_items_synced}")
print(f"Tags created: {result.total_tags_created}")
print(f"Edges created: {result.total_edges_created}")
print(f"Duration: {result.duration_seconds:.1f}s")

# Access per-source results
for source, src_result in result.source_results.items():
    print(f"  {source}: {src_result.items_synced} items, {src_result.tags_created} tags")

# Sync individual sources
git_result = orchestrator.sync_git(since_days=30, use_blame=True)
issue_result = orchestrator.sync_issues(
    provider="github", repo="owner/repo", token=token
)
sentry_result = orchestrator.sync_sentry(
    org_slug="org", project_slug="proj", token=token
)

# Get sync statistics
stats = orchestrator.get_sync_stats()

# Sync git history
python -m src.services.external_context.orchestrator \
    --duckdb /path/to/cpg.duckdb \
    --repo-path /path/to/repo \
    --sync-git --git-since-days 30

# Sync git without blame analysis
python -m src.services.external_context.orchestrator \
    --duckdb /path/to/cpg.duckdb \
    --sync-git --no-blame

# Sync git with custom max commits
python -m src.services.external_context.orchestrator \
    --duckdb /path/to/cpg.duckdb \
    --sync-git --git-max-commits 2000

# Sync GitHub issues
python -m src.services.external_context.orchestrator \
    --duckdb /path/to/cpg.duckdb \
    --sync-issues --issue-provider github --issue-repo owner/repo \
    --issue-token "$GITHUB_TOKEN"

# Sync GitLab issues (self-hosted)
python -m src.services.external_context.orchestrator \
    --duckdb /path/to/cpg.duckdb \
    --sync-issues --issue-provider gitlab --issue-repo owner/repo \
    --issue-url https://gitlab.company.com \
    --issue-token "$GITLAB_TOKEN"

# Sync Jira issues
python -m src.services.external_context.orchestrator \
    --duckdb /path/to/cpg.duckdb \
    --sync-issues --issue-provider jira --issue-project PROJ \
    --issue-url https://company.atlassian.net \
    --issue-token "$JIRA_TOKEN"

# Sync Sentry errors
python -m src.services.external_context.orchestrator \
    --duckdb /path/to/cpg.duckdb \
    --sync-sentry --sentry-org my-org --sentry-project my-project \
    --sentry-token "$SENTRY_AUTH_TOKEN"

# Sync Sentry (self-hosted)
python -m src.services.external_context.orchestrator \
    --duckdb /path/to/cpg.duckdb \
    --sync-sentry --sentry-org my-org --sentry-project my-project \
    --sentry-token "$SENTRY_AUTH_TOKEN" --sentry-url https://sentry.company.com

# Sync all sources
python -m src.services.external_context.orchestrator \
    --duckdb /path/to/cpg.duckdb \
    --repo-path /path/to/repo \
    --sync-all

# View sync statistics
python -m src.services.external_context.orchestrator \
    --duckdb /path/to/cpg.duckdb --stats

# Verbose output
python -m src.services.external_context.orchestrator \
    --duckdb /path/to/cpg.duckdb --sync-git --verbose

from src.services.external_context import (
    sync_git_to_cpg,
    sync_issues_to_cpg,
    sync_sentry_to_cpg,
)

# Sync git metadata
result = sync_git_to_cpg(
    duckdb_conn=conn,
    pg_conn=pg_conn,       # optional
    repo_path=".",
    since_days=30,
    use_blame=True
)

# Sync issues
result = sync_issues_to_cpg(
    duckdb_conn=conn,
    pg_conn=pg_conn,
    provider="github",
    repo="owner/repo",
    token=os.getenv("GITHUB_TOKEN")
)

# Sync Sentry errors
result = sync_sentry_to_cpg(
    duckdb_conn=conn,
    pg_conn=pg_conn,
    org_slug="my-org",
    project_slug="my-project",
    token=os.getenv("SENTRY_AUTH_TOKEN")
)

class ExternalContextBase(ABC):
    def __init__(self, duckdb_conn, pg_conn=None, source: ExternalSource = None):
        ...

# Verify git history is accessible
python -m src.services.external_context.orchestrator \
    --duckdb /path/to/cpg.duckdb \
    --sync-git --git-since-days 7 --verbose

# Skip blame if it's too slow on large repos
python -m src.services.external_context.orchestrator \
    --duckdb /path/to/cpg.duckdb \
    --sync-git --no-blame

service = IssueSyncService(
    duckdb_conn=conn,
    provider="github",
    repo="owner/repo",
    token=os.getenv("GITHUB_TOKEN")
)

-- Inspect tag values
SELECT name, value, COUNT(*) FROM nodes_tag_v2 GROUP BY name, value LIMIT 20;

-- Check edge count
SELECT COUNT(*) FROM edges_tagged_by;