middleware/platformscrape.ts

--- title: "Route Flow: ssutil /src/middleware/platformscrape.ts " --- flowchart TD subgraph GET ["GET"] hono_get_content_type["GET content-type"] style hono_get_content_type fill:#5E377B,color:#fff,stroke:#B4976A hono_get_key["GET key"] style hono_get_key fill:#5E377B,color:#fff,stroke:#B4976A hono_get_userKey["GET userKey"] style hono_get_userKey fill:#5E377B,color:#fff,stroke:#B4976A end subgraph PUT ["PUT"] hono_put_key["PUT key"] style hono_put_key fill:#5E377B,color:#fff,stroke:#B4976A hono_put_key["PUT key"] style hono_put_key fill:#5E377B,color:#fff,stroke:#B4976A hono_put_key["PUT key"] style hono_put_key fill:#5E377B,color:#fff,stroke:#B4976A end normalizeUrl{{"normalizeUrl"}} style normalizeUrl fill:#161b22,color:#fff,stroke:#c2a35d normalizeDomain{{"normalizeDomain"}} style normalizeDomain fill:#161b22,color:#fff,stroke:#c2a35d domainToId{{"domainToId"}} style domainToId fill:#161b22,color:#fff,stroke:#c2a35d constructFullAddress{{"constructFullAddress"}} style constructFullAddress fill:#161b22,color:#fff,stroke:#c2a35d setIdFromAccount{{"setIdFromAccount"}} style setIdFromAccount fill:#161b22,color:#fff,stroke:#c2a35d setIdFromDomain{{"setIdFromDomain"}} style setIdFromDomain fill:#161b22,color:#fff,stroke:#c2a35d extractWithChatGPTHelper{{"extractWithChatGPTHelper"}} style extractWithChatGPTHelper fill:#161b22,color:#fff,stroke:#c2a35d validateLogoUrl{{"validateLogoUrl"}} style validateLogoUrl fill:#161b22,color:#fff,stroke:#c2a35d detectPlatform{{"detectPlatform"}} style detectPlatform fill:#161b22,color:#fff,stroke:#c2a35d scrapeDealer{{"scrapeDealer"}} style scrapeDealer fill:#161b22,color:#fff,stroke:#c2a35d platformscrapeMiddleware{{"platformscrapeMiddleware"}} style platformscrapeMiddleware fill:#161b22,color:#fff,stroke:#c2a35d domainToId --> normalizeDomain scrapeDealer --> normalizeDomain scrapeDealer --> validateLogoUrl scrapeDealer --> constructFullAddress scrapeDealer --> detectPlatform scrapeDealer --> extractWithChatGPTHelper scrapeDealer --> setIdFromAccount scrapeDealer --> setIdFromDomain platformscrapeMiddleware --> normalizeUrl platformscrapeMiddleware --> scrapeDealer linkStyle default stroke:#c2a35d,stroke-width:2px,color:#fff classDef default color:#fff

--- title: "AST Structure: ssutil /src/middleware/platformscrape.ts " --- %%{init: { 'flowchart': { 'nodeSpacing': 20, 'rankSpacing': 30 }, 'themeVariables': { 'fontSize': '11px' } } }%% graph LR n0["SF"] n2["hono"] style n2 fill:#ffffff10,stroke:#ffffff33,color:#a1a1a1 n3["ID: Context"] n0 --> n3 n3 --> n2 style n3 stroke-dasharray:5,stroke:#B4976A n4["ID: Next"] n0 --> n4 n4 --> n2 style n4 stroke-dasharray:5,stroke:#B4976A n6["./chatgpt-extract"] style n6 fill:#ffffff10,stroke:#ffffff33,color:#a1a1a1 style n6 cursor:pointer,stroke-width:2px,stroke:#B4976A n7["ID: extractWithChatGPT"] n0 --> n7 n7 --> n6 style n7 stroke-dasharray:5,stroke:#B4976A n9["./domain-to-id"] style n9 fill:#ffffff10,stroke:#ffffff33,color:#a1a1a1 style n9 cursor:pointer,stroke-width:2px,stroke:#B4976A n10["ID: generateUniqueDomainId"] n0 --> n10 n10 --> n9 style n10 stroke-dasharray:5,stroke:#B4976A n12["./auth"] style n12 fill:#ffffff10,stroke:#ffffff33,color:#a1a1a1 style n12 cursor:pointer,stroke-width:2px,stroke:#B4976A n13["ID: UserIDToAPIKey"] n0 --> n13 n13 --> n12 style n13 stroke-dasharray:5,stroke:#B4976A n14["ID: domaintopassword"] n0 --> n14 n14 --> n12 style n14 stroke-dasharray:5,stroke:#B4976A n15["ID: saltpassword"] n0 --> n15 n15 --> n12 style n15 stroke-dasharray:5,stroke:#B4976A n17["./platformextractors/dealercom"] style n17 fill:#ffffff10,stroke:#ffffff33,color:#a1a1a1 style n17 cursor:pointer,stroke-width:2px,stroke:#B4976A n18["ID: DDC_PATTERNS"] n0 --> n18 n18 --> n17 style n18 stroke-dasharray:5,stroke:#B4976A n19["ID: extractDealerCom"] n0 --> n19 n19 --> n17 style n19 stroke-dasharray:5,stroke:#B4976A n21["./platformextractors/dealercenter"] style n21 fill:#ffffff10,stroke:#ffffff33,color:#a1a1a1 style n21 cursor:pointer,stroke-width:2px,stroke:#B4976A n22["ID: DEALERCENTER_PATTERNS"] n0 --> n22 n22 --> n21 style n22 stroke-dasharray:5,stroke:#B4976A n23["ID: extractDealerCenter"] n0 --> n23 n23 --> n21 style n23 stroke-dasharray:5,stroke:#B4976A n25["./platformextractors/overfuel"] style n25 fill:#ffffff10,stroke:#ffffff33,color:#a1a1a1 style n25 cursor:pointer,stroke-width:2px,stroke:#B4976A n26["ID: OVERFUEL_PATTERNS"] n0 --> n26 n26 --> n25 style n26 stroke-dasharray:5,stroke:#B4976A n27["ID: extractOverfuel"] n0 --> n27 n27 --> n25 style n27 stroke-dasharray:5,stroke:#B4976A n29["./platformextractors/dealeron"] style n29 fill:#ffffff10,stroke:#ffffff33,color:#a1a1a1 style n29 cursor:pointer,stroke-width:2px,stroke:#B4976A n30["ID: DEALERON_PATTERNS"] n0 --> n30 n30 --> n29 style n30 stroke-dasharray:5,stroke:#B4976A n31["ID: extractDealerOn"] n0 --> n31 n31 --> n29 style n31 stroke-dasharray:5,stroke:#B4976A n33["./platformextractors/dealerfire"] style n33 fill:#ffffff10,stroke:#ffffff33,color:#a1a1a1 style n33 cursor:pointer,stroke-width:2px,stroke:#B4976A n34["ID: DEALERFIRE_PATTERNS"] n0 --> n34 n34 --> n33 style n34 stroke-dasharray:5,stroke:#B4976A n35["ID: extractDealerFire"] n0 --> n35 n35 --> n33 style n35 stroke-dasharray:5,stroke:#B4976A n37["./platformextractors/dealerinspire"] style n37 fill:#ffffff10,stroke:#ffffff33,color:#a1a1a1 style n37 cursor:pointer,stroke-width:2px,stroke:#B4976A n38["ID: DEALERINSPIRE_PATTERNS"] n0 --> n38 n38 --> n37 style n38 stroke-dasharray:5,stroke:#B4976A n39["ID: extractDealerInspire"] n0 --> n39 n39 --> n37 style n39 stroke-dasharray:5,stroke:#B4976A n259["FD: normalizeUrl"] n0 --> n259 n355["FD: normalizeDomain"] n0 --> n355 n371["FD: domainToId"] n0 --> n371 n411["FD: constructFullAddress"] n0 --> n411 n754["FD: setIdFromAccount"] n0 --> n754 n829["FD: setIdFromDomain"] n0 --> n829 n918["FD: extractWithChatGPTHelper"] n0 --> n918 subgraph sub957 ["Safe Boundary"] end n1104["FD: validateLogoUrl"] n0 --> n1104 subgraph sub1115 ["Safe Boundary"] end n1153["FD: detectPlatform"] n0 --> n1153 n1264["FD: scrapeDealer"] n0 --> n1264 subgraph sub1295 ["Safe Boundary"] end subgraph sub1316 ["Safe Boundary"] subgraph sub1337 ["Safe Boundary"] subgraph sub1551 ["Safe Boundary"] end subgraph sub1580 ["Safe Boundary"] end subgraph sub1597 ["Safe Boundary"] subgraph sub1615 ["Safe Boundary"] end end end end subgraph sub1720 ["Safe Boundary"] subgraph sub2283 ["Safe Boundary"] end subgraph sub2500 ["Safe Boundary"] end end subgraph sub2731 ["Safe Boundary"] end n2894["CE: c.json(400)"] n0 --> n2894 n2912["CE: c.json(400)"] n0 --> n2912 n2930["CE: c.json(result)"] n0 --> n2930 n2967["CE: c.json(400)"] n0 --> n2967 n2987["CE: c.json(400)"] n0 --> n2987 n3008["CE: c.json(result)"] n0 --> n3008 n3014["CE: next"] n0 --> n3014 style n0 fill:#5E377B,color:#fff,stroke:#B4976A linkStyle default stroke:#c2a35d,stroke-width:1px,opacity:0.6 click n6 "../middleware/chatgpt-extract.html" "View Documentation" click n9 "../middleware/domain-to-id.html" "View Documentation" click n12 "../middleware/auth.html" "View Documentation" click n17 "../middleware/platformextractors/dealercom.html" "View Documentation" click n21 "../middleware/platformextractors/dealercenter.html" "View Documentation" click n25 "../middleware/platformextractors/overfuel.html" "View Documentation" click n29 "../middleware/platformextractors/dealeron.html" "View Documentation" click n33 "../middleware/platformextractors/dealerfire.html" "View Documentation" click n37 "../middleware/platformextractors/dealerinspire.html" "View Documentation"