Graffold — Architecture Diagrams

1. Knowledge Graph Ingestion Pipeline

flowchart TB
    %% ── Entry Points ───────────────────────────────────────────
    subgraph ENTRY["Entry Points"]
        PM["pipeline_main.py<br/>Full automated pipeline"]
        EM["enrichment_main.py<br/>CSV / Parquet / TXT"]
        IM["ingest_main.py<br/>PubMed abstracts"]
    end

    PM --> STEP1 & STEP2 & STEP3 & STEP4 & STEP5 & STEP6

    %% ── Pipeline Steps ─────────────────────────────────────────
    subgraph PIPELINE["PipelineExecutor — 6 Steps"]
        direction TB

        subgraph STEP1["Step 1 · CSV Foundation"]
            direction TB
            S1A["1a — OBO ontology files<br/>(MONDO disease hierarchy)"]
            S1B["1b — Disease ontology CSVs"]
            S1C["1c — Protein dictionaries<br/>(UniProt IDs)"]
            S1D["1d — Cell type data"]
            S1E["1e — Other CSVs"]
        end

        subgraph STEP2["Step 2 · PubMed Ingestion"]
            direction TB
            FETCH["Fetch abstracts<br/>via Entrez API"]
            CHUNK["Chunk text<br/>(overlapping windows)"]
            EXTRACT["LLM entity/relationship<br/>extraction per chunk"]
            STORE_KG["Store nodes + rels<br/>in graph"]
        end

        subgraph STEP3["Step 3 · Entity Consolidation"]
            direction TB
            EC_UNI["UniProt ID merging"]
            EC_SYN["Synonym / name-feature<br/>deduplication"]
            EC_FUZZY["Fuzzy name matching"]
        end

        subgraph STEP4["Step 4 · Relationship Consolidation"]
            direction TB
            RC_DUP["Merge duplicate rels"]
            RC_FREQ["Aggregate frequency,<br/>PMIDs, confidence"]
        end

        subgraph STEP5["Step 5 · Embeddings"]
            direction TB
            EMB_NODE["Node embeddings<br/>(all-distilroberta-v1)"]
            EMB_CHUNK["Chunk embeddings<br/>(all-distilroberta-v1)"]
            IDX["Create vector + fulltext<br/>indexes"]
        end

        subgraph STEP6["Step 6 · Validation"]
            VALIDATE["Node/rel counts,<br/>index health,<br/>quality report"]
        end

        STEP1 --> STEP2 --> STEP3 --> STEP4 --> STEP5 --> STEP6
    end

    %% ── Supporting Components ──────────────────────────────────
    subgraph PROCESSORS["Processors"]
        direction LR
        ENR_ORCH["EnrichmentOrchestrator<br/>CSV column analysis +<br/>auto-handler routing"]
        OBO_PROC["OBOStructureProcessor<br/>ontology → graph"]
        BIO_ENR["BiologicalEnrichment<br/>GO terms, pathways,<br/>subcellular location"]
        ONT_FILT["OntologyFilter<br/>MONDO / UniProt<br/>standardization"]
    end

    subgraph KG_PIPE["KGPipeline (LangGraph)"]
        direction TB
        KG_LLM["LLM extraction<br/>(Ollama / Bedrock / OpenAI)"]
        KG_UNIPROT["UniProt ID resolution"]
        KG_STORE["Cypher MERGE<br/>nodes + relationships"]
    end

    subgraph ENTITY_RES["EntityResolver"]
        direction LR
        ER_FULL["run_full_entity_resolution()"]
    end

    subgraph REL_COUNT["RelationshipCounter"]
        direction LR
        RC_FULL["consolidate_comprehensive<br/>_relationships()"]
    end

    subgraph EMB_PIPE["EmbeddingPipeline"]
        direction LR
        EP_ADD["add_embeddings_to_kg()"]
        EP_IDX["create vector + fulltext<br/>indexes"]
    end

    %% ── Database ───────────────────────────────────────────────
    NEO4J[("Graph Database<br/>(Neo4j / Memgraph)")]

    %% ── Connections ────────────────────────────────────────────
    EM --> ENR_ORCH
    IM --> KG_PIPE

    S1A --> OBO_PROC
    S1B & S1C & S1D & S1E --> ENR_ORCH
    ENR_ORCH --> KG_PIPE
    OBO_PROC --> NEO4J

    STEP2 -.-> KG_PIPE
    FETCH --> CHUNK --> EXTRACT --> STORE_KG
    KG_LLM --> KG_UNIPROT --> KG_STORE
    KG_STORE --> NEO4J

    STEP3 -.-> ENTITY_RES
    EC_UNI & EC_SYN & EC_FUZZY -.-> ER_FULL
    ER_FULL --> NEO4J

    STEP4 -.-> REL_COUNT
    RC_DUP & RC_FREQ -.-> RC_FULL
    RC_FULL --> NEO4J

    STEP5 -.-> EMB_PIPE
    EMB_NODE & EMB_CHUNK -.-> EP_ADD
    IDX -.-> EP_IDX
    EP_ADD & EP_IDX --> NEO4J

    STEP6 --> NEO4J

    BIO_ENR & ONT_FILT -.-> KG_PIPE

    %% ── Factories ──────────────────────────────────────────────
    subgraph FACTORIES["Factories"]
        direction LR
        LLM_F["LLMFactory<br/>Ollama / Bedrock /<br/>SageMaker / OpenAI"]
        EMB_F["EmbeddingFactory<br/>sentence-transformers"]
    end

    LLM_F -.-> KG_LLM & EXTRACT
    EMB_F -.-> EMB_PIPE

    %% ── Styling ────────────────────────────────────────────────
    classDef entryStyle fill:#4A90D9,stroke:#2C5F8A,color:#fff
    classDef stepStyle fill:#50C878,stroke:#3A9D5C,color:#fff
    classDef procStyle fill:#7B68EE,stroke:#5A4FCF,color:#fff
    classDef dbStyle fill:#FF8C42,stroke:#CC6F35,color:#fff
    classDef factoryStyle fill:#A0A0A0,stroke:#707070,color:#fff

    class PM,EM,IM entryStyle
    class S1A,S1B,S1C,S1D,S1E,FETCH,CHUNK,EXTRACT,STORE_KG stepStyle
    class EC_UNI,EC_SYN,EC_FUZZY,RC_DUP,RC_FREQ stepStyle
    class EMB_NODE,EMB_CHUNK,IDX,VALIDATE stepStyle
    class ENR_ORCH,OBO_PROC,BIO_ENR,ONT_FILT,KG_LLM,KG_UNIPROT,KG_STORE procStyle
    class ER_FULL,RC_FULL,EP_ADD,EP_IDX procStyle
    class NEO4J dbStyle
    class LLM_F,EMB_F factoryStyle

---

2. API + Query Agents

flowchart TB
    %% ── Client ─────────────────────────────────────────────────
    CLIENT(["Client<br/>(Streamlit UI / REST)"])

    %% ── API Layer ──────────────────────────────────────────────
    subgraph API["FastAPI"]
        direction TB
        EP_CREATE["POST /v1/sessions<br/>→ create session + agent"]
        EP_QUERY["POST /v1/sessions/{id}/query<br/>→ execute query"]
        EP_KNN["POST /v1/sessions/{id}/expand-knn"]
        GUARDS["SecurityGuardrails<br/>PII detection · injection check"]
    end

    %% ── Session Manager ────────────────────────────────────────
    subgraph SM["SessionManager"]
        SM_CREATE["create_session(config)<br/>→ QueryService.create_agent()"]
        SM_EXEC["execute_query()<br/>→ restore history<br/>→ QueryService.execute_query()"]
        SM_HIST["QuerySession<br/>agent + config + history"]
    end

    %% ── Query Service ──────────────────────────────────────────
    subgraph QS["QueryService (Orchestrator)"]
        QS_AGENT["create_agent(config)<br/>routes by agent_type"]
        QS_EXEC["execute_query()<br/>→ agent.sync_query()"]
        QS_REFRAG["REFRAG compression<br/>(optional)"]
        QS_SAN["sanitize_result()"]
    end

    %% ── Agent Selection ────────────────────────────────────────
    subgraph AGENTS["Agent Layer"]
        direction LR
        NEO4J_A["Neo4jQueryAgent<br/>(Standard / Hybrid)"]
        DYN_A["DynamicQueryAgent<br/>(Dynamic)"]
        LANG_A["LangGraphAgent<br/>(Agentic)"]
    end

    %% ── Neo4jQueryAgent detail ─────────────────────────────────
    subgraph NEO4J_DETAIL["Neo4jQueryAgent (extends Dynamic)"]
        direction TB
        N_HYBRID["HybridRetriever<br/>vector + fulltext seed search"]
        N_NEIGH["Neighborhood retrieval<br/>Cypher graph traversal"]
        N_GRAPHRAG["GraphRAG<br/>retriever + LLM generation"]
        N_KNN["APOC KNN expansion"]
    end

    %% ── DynamicQueryAgent detail ───────────────────────────────
    subgraph DYN_DETAIL["DynamicQueryAgent Pipeline"]
        direction TB
        D_QP["QueryProcessor<br/>rewrite + expand query"]
        D_ED["EntityDiscovery<br/>regex extraction"]
        D_SS["6 Search Strategies"]
        D_RP["ResultProcessor<br/>CrossEncoder re-ranking"]
        D_LLM["LLM answer synthesis"]
    end

    subgraph STRATEGIES["Search Strategies (parallel)"]
        direction LR
        S1["Vector<br/>similarity"]
        S2["Fulltext<br/>search"]
        S3["Schema-guided<br/>expansion"]
        S4["Literature<br/>search"]
        S5["Text2Cypher<br/>(LLM → Cypher)"]
        S6["Dynamic<br/>expansion"]
    end

    %% ── LangGraphAgent detail ──────────────────────────────────
    subgraph LANG_DETAIL["LangGraphAgent (ReAct)"]
        direction TB
        L_REACT["create_react_agent<br/>+ MemorySaver"]
        subgraph TOOLS["Tools (called iteratively)"]
            direction LR
            T1["count_nodes"]
            T2["lookup_entity"]
            T3["search_knowledge_graph"]
        end
        L_CONF["_assess_confidence()"]
    end

    %% ── REFRAG ─────────────────────────────────────────────────
    subgraph REFRAG["REFRAG Processor"]
        direction LR
        R_ENC["ChunkEncoder<br/>16-token → dense embedding"]
        R_SEL["CriticalitySelector<br/>preserve key chunks"]
    end

    %% ── Core Layer ─────────────────────────────────────────────
    subgraph CORE["Core Layer"]
        direction LR
        DB["DatabaseInterface<br/>(Neo4j / Memgraph)"]
        VR["VectorRetriever<br/>chunk_embeddings"]
        HR["HybridRetriever<br/>node_embeddings +<br/>node_fulltext"]
        T2CR["Text2CypherRetriever"]
    end

    NEO4J[("Neo4j<br/>Knowledge Graph")]

    %% ── Factories ──────────────────────────────────────────────
    subgraph FACTORIES["Factories"]
        direction LR
        LLM_F["LLMFactory<br/>Ollama / Bedrock /<br/>SageMaker / OpenAI"]
        EMB_F["EmbeddingFactory<br/>sentence-transformers"]
    end

    %% ── Connections ────────────────────────────────────────────

    CLIENT --> EP_CREATE & EP_QUERY & EP_KNN
    EP_CREATE & EP_QUERY --> GUARDS
    GUARDS --> SM_CREATE & SM_EXEC

    SM_CREATE --> QS_AGENT
    SM_EXEC --> QS_EXEC
    QS_EXEC --> QS_REFRAG --> QS_SAN

    QS_AGENT -->|"Standard / Hybrid"| NEO4J_A
    QS_AGENT -->|"Dynamic"| DYN_A
    QS_AGENT -->|"Agentic"| LANG_A

    NEO4J_A -.->|extends| DYN_A

    %% Neo4j agent flow
    NEO4J_A --> NEO4J_DETAIL
    N_HYBRID --> N_NEIGH --> N_GRAPHRAG
    N_KNN -.->|"optional"| N_NEIGH

    %% Dynamic agent flow
    DYN_A --> DYN_DETAIL
    D_QP --> D_ED --> D_SS
    D_SS --> STRATEGIES
    S1 & S2 & S3 & S4 & S5 & S6 --> D_RP --> D_LLM

    %% LangGraph agent flow
    LANG_A --> LANG_DETAIL
    L_REACT --> TOOLS
    TOOLS --> L_CONF

    %% REFRAG
    QS_REFRAG -.-> REFRAG
    R_ENC --> R_SEL

    %% All agents → core → DB
    NEO4J_DETAIL --> VR & HR & T2CR
    STRATEGIES --> VR & HR & T2CR
    T1 & T2 & T3 --> DB
    VR & HR & T2CR --> DB
    DB --> NEO4J

    %% Factories
    LLM_F -.-> DYN_A & LANG_A & NEO4J_A
    EMB_F -.-> DYN_A & LANG_A & NEO4J_A

    %% ── Styling ────────────────────────────────────────────────
    classDef clientStyle fill:#E8E8E8,stroke:#999,color:#333
    classDef apiStyle fill:#4A90D9,stroke:#2C5F8A,color:#fff
    classDef serviceStyle fill:#7B68EE,stroke:#5A4FCF,color:#fff
    classDef agentStyle fill:#50C878,stroke:#3A9D5C,color:#fff
    classDef toolStyle fill:#FFD700,stroke:#B8A000,color:#333
    classDef coreStyle fill:#FF8C42,stroke:#CC6F35,color:#fff
    classDef dbStyle fill:#FF8C42,stroke:#CC6F35,color:#fff
    classDef factoryStyle fill:#A0A0A0,stroke:#707070,color:#fff

    class CLIENT clientStyle
    class EP_CREATE,EP_QUERY,EP_KNN,GUARDS apiStyle
    class SM_CREATE,SM_EXEC,SM_HIST,QS_AGENT,QS_EXEC,QS_REFRAG,QS_SAN serviceStyle
    class NEO4J_A,DYN_A,LANG_A agentStyle
    class T1,T2,T3,L_REACT,L_CONF toolStyle
    class N_HYBRID,N_NEIGH,N_GRAPHRAG,N_KNN coreStyle
    class D_QP,D_ED,D_SS,D_RP,D_LLM coreStyle
    class S1,S2,S3,S4,S5,S6 coreStyle
    class R_ENC,R_SEL coreStyle
    class DB,VR,HR,T2CR coreStyle
    class NEO4J dbStyle
    class LLM_F,EMB_F factoryStyle

---

Agent Selection

Database Abstraction: All components access the graph database through a DatabaseInterface abstraction layer. The concrete backend (Neo4j, Memgraph, or FalkorDB) is selected at runtime via the DATABASE_TYPE environment variable. This makes backends fully swappable without code changes.

agent_type	Agent Class	Approach
Standard (default)	Neo4jQueryAgent	GraphRAG + hybrid retrieval + KNN expansion
Hybrid	Neo4jQueryAgent	Same as Standard
Dynamic	DynamicQueryAgent	6-strategy parallel search + CrossEncoder re-ranking
Agentic	LangGraphAgent	ReAct loop with iterative tool calling

Note: Despite the Neo4j prefix in class names, all agents work with Neo4j, Memgraph, and FalkorDB backends via the DatabaseInterface abstraction. The backend is selected via the DATABASE_TYPE environment variable.