From 85310ada9dfd999ee085aa2b827d9545cbd38deb Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 26 Jun 2026 08:48:24 -0300 Subject: [PATCH 01/65] init subdomain-takeover + NATS skeleton --- scanners/subdomain-takeover/.env.example | 7 + scanners/subdomain-takeover/Dockerfile | 22 ++ scanners/subdomain-takeover/README.md | 137 ++++++++++++ .../subdomain-takeover/cmd/service/main.go | 92 ++++++++ scanners/subdomain-takeover/config.go | 37 ++++ .../fixtures/cyber-gc-ca_dns_scan.json | 202 ++++++++++++++++++ .../fixtures/www-cyber-gc-ca_dns_scan.json | 23 ++ scanners/subdomain-takeover/go.mod | 19 ++ scanners/subdomain-takeover/go.sum | 21 ++ scanners/subdomain-takeover/model/finding.go | 0 scanners/subdomain-takeover/model/types.go | 0 scanners/subdomain-takeover/nats/consumer.go | 0 scanners/subdomain-takeover/nats/publisher.go | 0 .../subdomain-takeover/scanner/confidence.go | 0 .../subdomain-takeover/scanner/dns_checks.go | 0 .../subdomain-takeover/scanner/scanner.go | 0 16 files changed, 560 insertions(+) create mode 100644 scanners/subdomain-takeover/.env.example create mode 100644 scanners/subdomain-takeover/Dockerfile create mode 100644 scanners/subdomain-takeover/README.md create mode 100644 scanners/subdomain-takeover/cmd/service/main.go create mode 100644 scanners/subdomain-takeover/config.go create mode 100644 scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json create mode 100644 scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json create mode 100644 scanners/subdomain-takeover/go.mod create mode 100644 scanners/subdomain-takeover/go.sum create mode 100644 scanners/subdomain-takeover/model/finding.go create mode 100644 scanners/subdomain-takeover/model/types.go create mode 100644 scanners/subdomain-takeover/nats/consumer.go create mode 100644 scanners/subdomain-takeover/nats/publisher.go create mode 100644 scanners/subdomain-takeover/scanner/confidence.go create mode 100644 scanners/subdomain-takeover/scanner/dns_checks.go create mode 100644 scanners/subdomain-takeover/scanner/scanner.go diff --git a/scanners/subdomain-takeover/.env.example b/scanners/subdomain-takeover/.env.example new file mode 100644 index 0000000000..31d2f06813 --- /dev/null +++ b/scanners/subdomain-takeover/.env.example @@ -0,0 +1,7 @@ +NATS_URL= +NATS_STREAM= +SUBJECT_IN= +SUBJECT_OUT= +DURABLE_NAME= +WORKER_COUNT= +LOG_LEVEL= diff --git a/scanners/subdomain-takeover/Dockerfile b/scanners/subdomain-takeover/Dockerfile new file mode 100644 index 0000000000..6c7f3eafaa --- /dev/null +++ b/scanners/subdomain-takeover/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.25-alpine AS builder + +WORKDIR /src +RUN apk add --no-cache git ca-certificates + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/subdomain-takeover ./cmd/service + +# Runtime stage +FROM alpine:3.22 + +RUN addgroup -S scanner && adduser -S scanner -G scanner +WORKDIR /app +RUN apk add --no-cache ca-certificates tzdata + +COPY --from=builder /out/subdomain-takeover /app/subdomain-takeover + +USER scanner +ENTRYPOINT ["/app/subdomain-takeover"] diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md new file mode 100644 index 0000000000..0bfcdaf9ba --- /dev/null +++ b/scanners/subdomain-takeover/README.md @@ -0,0 +1,137 @@ +# subdomain-takeover-service + +Go microservice that consumes DNS scan results and emits normalized subdomain takeover findings. + +## What it does + +1. Consumes from `scans.dns_scanner_results` (JetStream stream: `SCANS`) +2. Evaluates takeover risk using passive DNS evidence + provider fingerprints +3. Assigns confidence (`suspected`, `probable`, `confirmed`) +4. Publishes findings to `scans.findings.upsert` + +## Passive-only detection policy + +This service uses passive checks only. +It does not attempt account takeover, resource claiming, or any active validation. + +As a result, confidence is conservative and evidence-driven: +- `suspected`: weak or incomplete takeover indicators +- `probable`: strong passive indicators with known exploitable conditions +- `confirmed`: reserved for deterministic passive signatures only + +## Sitting Duck DNS hijack coverage + +The scanner includes passive detection for Sitting Duck style DNS hijack risk. + +A Sitting Duck takeover is considered possible when all of the following are true: + +1. The registered domain uses or delegates authoritative DNS services to a provider other than the registrar. +2. The authoritative name server cannot resolve authoritatively for the domain (lame delegation). +3. The DNS provider is known to allow domain/zone claiming without strong ownership verification. + +### Variations covered + +- Partially lame delegation: some NS records are lame, others still answer. +- Full lame delegation: all authoritative NS are lame. +- Redelegation drift: domain appears redelegated while stale or invalid NS remains and creates takeover exposure. + +If lame delegation and exploitable provider conditions are both present, the domain is considered hijackable risk. + +## Confidence model + +This model applies across both CNAME-based takeover checks and NS-based Sitting Duck checks. + +- `suspected` + - dangling/suspicious DNS pattern only, or + - lame delegation evidence without known exploitable-provider confirmation +- `probable` + - DNS pattern + known provider unclaimed signature, or + - lame delegation confirmed (`partial` or `full`) + exploitable-provider rule match +- `confirmed` + - explicit passive takeover indicator with deterministic evidence + - if deterministic passive proof is missing, cap at `probable` + +## Required evidence for findings + +Each finding should include: +- `domain` +- `domain_key` +- `record_type` (`CNAME` or `NS`) +- `target` or `ns_host` +- `provider` +- `lame_type` (`partial` or `full`) for Sitting Duck findings +- per-NS resolver outcomes where applicable +- `confidence` +- `reason_code` +- remediation guidance + +## Remediation guidance + +Typical remediation: +1. Fix delegation at registrar and authoritative DNS provider. +2. Remove stale or dead NS entries. +3. Reclaim or securely rebind abandoned DNS zones or accounts. +4. Validate ownership controls for DNS provider account workflows. + +## Prerequisites + +- Go 1.24+ +- NATS with JetStream enabled +- `nats` CLI (optional, for local testing) + +## Quick start + +```bash +go mod tidy +go run ./cmd/service +``` + +Environment variables: +- `NATS_URL` (default: `nats://localhost:4222`) +- `NATS_STREAM` (default: `SCANS`) +- `SUBJECT_IN` (default: `scans.dns_scanner_results`) +- `SUBJECT_OUT` (default: `scans.findings.upsert`) +- `DURABLE_NAME` (default: `subdomain_takeover`) +- `WORKER_COUNT` (default: `5`) +- `LOG_LEVEL` (default: `info`) + +Local test flow: + +Publish a test request: +```bash +nats pub scans.dns_scanner_results '{"domain":"example.gc.ca","domain_key":"12345","shared_id":null,"results":{"record_exists":true,"rcode":"NOERROR","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_records":{"hostnames":["ns1.example-dns-provider.net"],"warnings":[]}}}' +``` + +Watch findings: +```bash +nats sub "scans.findings.upsert" +``` + +## Development milestones + +1. Build config + NATS consume/publish loop. +2. Add DNS evidence extraction. +3. Add provider fingerprint rule engine. +4. Add confidence scoring and remediation text. +5. Add tests for classification rules and message handling. + +--- + +## External Go modules to install + +Start minimal: + +```bash +go get github.com/nats-io/nats.go +go get github.com/miekg/dns +go get github.com/rs/zerolog +go get golang.org/x/sync/errgroup +go get github.com/stretchr/testify +``` + +What each is for: +- `nats.go`: JetStream consume/publish +- `miekg/dns`: robust DNS lookups and record parsing +- `zerolog`: structured logs without heavy setup +- `errgroup`: clean goroutine lifecycle management +- `testify`: easier assertions in unit tests diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go new file mode 100644 index 0000000000..175a930507 --- /dev/null +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +func main() { + config := initConfig() + logger := config.Logger + ctx, cancel := context.WithCancel(context.Background()) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + + go func() { + <-sig + logger.Info().Msg("Shutdown requested...") + cancel() + }() + + nc, _ := nats.Connect(config.NatsUrl) + js, _ := jetstream.New(nc) + logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) + + s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ + Name: config.NatsStream, + Subjects: []string{config.SubjectIn, config.SubjectOut}, + }) + + cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: config.DurableName, + AckPolicy: jetstream.AckExplicitPolicy, + }) + + iter, _ := cons.Messages(jetstream.PullMaxMessages(1)) + var wg sync.WaitGroup + sem := make(chan struct{}, config.WorkerCount) + + for { + if nc.IsClosed() { + logger.Error().Msg("Connection to NATS is closed.") + break + } + + select { + case <-ctx.Done(): + return + default: + continue + } + + // sem <- struct{}{} + // go func() { + // defer func() { + // <-sem + // }() + // msg, err := iter.Next() + // if err != nil { + // logger.Error().Msgf("next err:", err.Error()) + // continue + // } + // fmt.Printf("Processing msg: %s\n", string(msg.Data())) + // // doWork() + // msg.Ack() + // }() + } + + waitDone := make(chan struct{}) + go func() { + wg.Wait() + close(waitDone) + }() + + select { + case <-waitDone: + logger.Info().Msg("all workers drained") + case <-time.After(30 * time.Second): + logger.Info().Msg("drain timeout") + } + + logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) + nc.Flush() + nc.Close() +} diff --git a/scanners/subdomain-takeover/config.go b/scanners/subdomain-takeover/config.go new file mode 100644 index 0000000000..b14c80710a --- /dev/null +++ b/scanners/subdomain-takeover/config.go @@ -0,0 +1,37 @@ +package main + +import ( + "os" + + "github.com/kelseyhightower/envconfig" + "github.com/rs/zerolog" +) + +type Config struct { + NatsUrl string `envconfig:"NATS_URL" default:"nats://localhost:4222"` + NatsStream string `envconfig:"NATS_STREAM" default:"SCANS"` + SubjectIn string `envconfig:"SUBJECT_IN" default:"scans.dns_scanner_results"` + SubjectOut string `envconfig:"SUBJECT_OUT" default:"scans.findings.upsert"` + DurableName string `envconfig:"DURABLE_NAME" default:"subdomain_takeover"` + WorkerCount int `envconfig:"WORKER_COUNT" default:"5"` + LogLevel zerolog.Level `envconfig:"LOG_LEVEL" default:"info"` + Logger zerolog.Logger +} + +func initConfig() *Config { + var cfg Config + if err := envconfig.Process("", &cfg); err != nil { + panic(err) + } + + zerolog.SetGlobalLevel(cfg.LogLevel) + + // Create multiple output steams for zerolog + multi := zerolog.MultiLevelWriter(zerolog.ConsoleWriter{Out: os.Stderr}) + + logger := zerolog.New(multi).With().Timestamp().Logger() + + cfg.Logger = logger + + return &cfg +} diff --git a/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json b/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json new file mode 100644 index 0000000000..cad720cc93 --- /dev/null +++ b/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json @@ -0,0 +1,202 @@ +{ + "domain": "cyber.gc.ca", + "base_domain": "cyber.gc.ca", + "zone_apex": "cyber.gc.ca", + "record_exists": true, + "rcode": "NOERROR", + "resolve_chain": [ + [ + "cyber.gc.ca. 7161 IN A 20.151.96.73" + ] + ], + "resolve_ips": [ + "20.151.96.73" + ], + "cname_record": null, + "mx_records": { + "hosts": [ + { + "preference": 10, + "hostname": "cyber-gc-ca.mail.protection.outlook.com", + "addresses": [ + "52.101.190.1", + "52.101.192.0", + "52.101.192.1", + "52.101.190.0" + ] + } + ], + "warnings": [] + }, + "ns_records": { + "hostnames": [ + "ns1.d-zone.ca", + "ns2.d-zone.ca", + "ns40.ent.global.gc.ca", + "ns41.ent.global.gc.ca", + "ns11.ent.global.gc.ca", + "ns10.ent.global.gc.ca" + ], + "warnings": [] + }, + "dkim": {}, + "spf": { + "record": "v=spf1 ip4:40.80.253.57 a:smtp.ops.cyber.gc.ca a:edge.cyber.gc.ca include:spf.protection.outlook.com -all", + "valid": true, + "dns_void_lookups": 0, + "warnings": [], + "parsed": { + "pass": [ + { + "value": "40.80.253.57", + "mechanism": "ip4" + }, + { + "value": "52.139.3.76", + "mechanism": "a" + }, + { + "value": "205.193.218.114", + "mechanism": "a" + }, + { + "value": "205.193.218.115", + "mechanism": "a" + } + ], + "neutral": [], + "softfail": [], + "fail": [], + "include": [ + { + "domain": "spf.protection.outlook.com", + "record": "v=spf1 ip4:40.92.0.0/15 ip4:40.107.0.0/16 ip4:52.100.0.0/15 ip4:52.102.0.0/16 ip4:52.103.0.0/17 ip4:104.47.0.0/17 ip6:2a01:111:f400::/48 ip6:2a01:111:f403::/49 ip6:2a01:111:f403:8000::/51 ip6:2a01:111:f403:c000::/51 ip6:2a01:111:f403:f000::/52 -all", + "dns_void_lookups": 0, + "parsed": { + "pass": [ + { + "value": "40.92.0.0/15", + "mechanism": "ip4" + }, + { + "value": "40.107.0.0/16", + "mechanism": "ip4" + }, + { + "value": "52.100.0.0/15", + "mechanism": "ip4" + }, + { + "value": "52.102.0.0/16", + "mechanism": "ip4" + }, + { + "value": "52.103.0.0/17", + "mechanism": "ip4" + }, + { + "value": "104.47.0.0/17", + "mechanism": "ip4" + }, + { + "value": "2a01:111:f400::/48", + "mechanism": "ip6" + }, + { + "value": "2a01:111:f403::/49", + "mechanism": "ip6" + }, + { + "value": "2a01:111:f403:8000::/51", + "mechanism": "ip6" + }, + { + "value": "2a01:111:f403:c000::/51", + "mechanism": "ip6" + }, + { + "value": "2a01:111:f403:f000::/52", + "mechanism": "ip6" + } + ], + "neutral": [], + "softfail": [], + "fail": [], + "include": [], + "redirect": null, + "exp": null, + "duplicate_include": [] + }, + "warnings": [], + "spf_default": "fail", + "lookups": 0 + } + ], + "redirect": null, + "exp": null, + "duplicate_include": [] + }, + "spf_default": "fail", + "lookups": 3 + }, + "dmarc": { + "record": "v=DMARC1; p=reject; pct=100; rua=mailto:dmarc@cyber.gc.ca", + "valid": true, + "location": "cyber.gc.ca", + "warnings": [], + "tags": { + "v": { + "value": "DMARC1", + "explicit": true + }, + "p": { + "value": "reject", + "explicit": true + }, + "pct": { + "value": 100, + "explicit": true + }, + "rua": { + "value": [ + { + "scheme": "mailto", + "address": "dmarc@cyber.gc.ca", + "size_limit": null, + "accepting": true + } + ], + "explicit": true + }, + "adkim": { + "value": "r", + "explicit": false + }, + "aspf": { + "value": "r", + "explicit": false + }, + "fo": { + "value": "0", + "explicit": false + }, + "rf": { + "value": "afrf", + "explicit": false + }, + "ri": { + "value": 86400, + "explicit": false + }, + "sp": { + "value": "reject", + "explicit": false + } + }, + "effective_policy_source": "p", + "effective_policy": "reject" + }, + "wildcard_sibling": false, + "wildcard_entry": false, + "zone_dnssec_enabled": null +} diff --git a/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json b/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json new file mode 100644 index 0000000000..8f546a74a5 --- /dev/null +++ b/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json @@ -0,0 +1,23 @@ +{ + "domain": "www.cyber.gc.ca", + "base_domain": "cyber.gc.ca", + "zone_apex": "www.cyber.gc.ca", + "record_exists": true, + "rcode": "NOERROR", + "resolve_chain": [ + ["www.cyber.gc.ca. 5550 IN CNAME cyber.gc.ca."], + ["cyber.gc.ca. 5550 IN A 20.151.96.73"] + ], + "cname_record": "www.cyber.gc.ca. 5550 IN CNAME cyber.gc.ca.", + "ns_records": { + "hostnames": [ + "ns40.ent.global.gc.ca", + "ns10.ent.global.gc.ca", + "ns41.ent.global.gc.ca", + "ns11.ent.global.gc.ca", + "ns2.d-zone.ca", + "ns1.d-zone.ca" + ], + "warnings": [] + } +} diff --git a/scanners/subdomain-takeover/go.mod b/scanners/subdomain-takeover/go.mod new file mode 100644 index 0000000000..f4bfb6b567 --- /dev/null +++ b/scanners/subdomain-takeover/go.mod @@ -0,0 +1,19 @@ +module github.com/canada-ca/tracker/scanners/subdomain-takeover + +go 1.25.0 + +require ( + github.com/kelseyhightower/envconfig v1.4.0 + github.com/rs/zerolog v1.35.1 +) + +require ( + github.com/klauspost/compress v1.18.5 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/nats-io/nats.go v1.52.0 // indirect + github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.49.0 // indirect + golang.org/x/sys v0.42.0 // indirect +) diff --git a/scanners/subdomain-takeover/go.sum b/scanners/subdomain-takeover/go.sum new file mode 100644 index 0000000000..470146821e --- /dev/null +++ b/scanners/subdomain-takeover/go.sum @@ -0,0 +1,21 @@ +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= +github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= +github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= +github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/scanners/subdomain-takeover/model/finding.go b/scanners/subdomain-takeover/model/finding.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/model/types.go b/scanners/subdomain-takeover/model/types.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/nats/consumer.go b/scanners/subdomain-takeover/nats/consumer.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/nats/publisher.go b/scanners/subdomain-takeover/nats/publisher.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/scanner/confidence.go b/scanners/subdomain-takeover/scanner/confidence.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/scanner/dns_checks.go b/scanners/subdomain-takeover/scanner/dns_checks.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/scanner/scanner.go b/scanners/subdomain-takeover/scanner/scanner.go new file mode 100644 index 0000000000..e69de29bb2 From 794cc25ac7db66544e6638fc2e2b00ff058312ae Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 26 Jun 2026 13:22:07 -0300 Subject: [PATCH 02/65] finish NATS consumer logic --- scanners/subdomain-takeover/main.go | 107 ++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 scanners/subdomain-takeover/main.go diff --git a/scanners/subdomain-takeover/main.go b/scanners/subdomain-takeover/main.go new file mode 100644 index 0000000000..102e03b573 --- /dev/null +++ b/scanners/subdomain-takeover/main.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +func main() { + config := initConfig() + logger := config.Logger + + ctx, cancel := context.WithCancel(context.Background()) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + + var wg sync.WaitGroup + sem := make(chan struct{}, config.WorkerCount) + + nc, _ := nats.Connect(config.NatsUrl) + js, _ := jetstream.New(nc) + logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) + + s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ + Name: config.NatsStream, + Subjects: []string{config.SubjectIn, config.SubjectOut}, + }) + + cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: config.DurableName, + AckPolicy: jetstream.AckExplicitPolicy, + }) + + iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) + if err != nil { + logger.Err(err) + } + + go func() { + <-sig + logger.Info().Msg("Shutdown requested...") + cancel() + iter.Stop() + }() + +Loop: + for { + if nc.IsClosed() { + logger.Error().Msg("Connection to NATS is closed.") + break Loop + } + + select { + case <-ctx.Done(): + break Loop + default: + } + + msg, err := iter.Next() + if err != nil { + if ctx.Err() != nil { + break Loop + } + logger.Debug().Err(err).Msg("next returned, continuing") + continue + } + + select { + case sem <- struct{}{}: + case <-ctx.Done(): + break Loop + } + + wg.Add(1) + go func(m jetstream.Msg) { + defer wg.Done() + defer func() { <-sem }() + fmt.Println(string(m.Data())) + m.Ack() + }(msg) + } + + waitDone := make(chan struct{}) + go func() { + wg.Wait() + close(waitDone) + }() + + select { + case <-waitDone: + logger.Info().Msg("all workers drained") + case <-time.After(30 * time.Second): + logger.Info().Msg("drain timeout") + } + + logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) + nc.Flush() + nc.Close() +} From c64e5e4c69eee06506c3e02a46e8f5fc4ede82fc Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 26 Jun 2026 13:24:14 -0300 Subject: [PATCH 03/65] move main.go to top dir --- .../subdomain-takeover/cmd/service/main.go | 92 ------------------- 1 file changed, 92 deletions(-) delete mode 100644 scanners/subdomain-takeover/cmd/service/main.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go deleted file mode 100644 index 175a930507..0000000000 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ /dev/null @@ -1,92 +0,0 @@ -package main - -import ( - "context" - "os" - "os/signal" - "sync" - "syscall" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" -) - -func main() { - config := initConfig() - logger := config.Logger - ctx, cancel := context.WithCancel(context.Background()) - - sig := make(chan os.Signal, 1) - signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) - - go func() { - <-sig - logger.Info().Msg("Shutdown requested...") - cancel() - }() - - nc, _ := nats.Connect(config.NatsUrl) - js, _ := jetstream.New(nc) - logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) - - s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ - Name: config.NatsStream, - Subjects: []string{config.SubjectIn, config.SubjectOut}, - }) - - cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ - Durable: config.DurableName, - AckPolicy: jetstream.AckExplicitPolicy, - }) - - iter, _ := cons.Messages(jetstream.PullMaxMessages(1)) - var wg sync.WaitGroup - sem := make(chan struct{}, config.WorkerCount) - - for { - if nc.IsClosed() { - logger.Error().Msg("Connection to NATS is closed.") - break - } - - select { - case <-ctx.Done(): - return - default: - continue - } - - // sem <- struct{}{} - // go func() { - // defer func() { - // <-sem - // }() - // msg, err := iter.Next() - // if err != nil { - // logger.Error().Msgf("next err:", err.Error()) - // continue - // } - // fmt.Printf("Processing msg: %s\n", string(msg.Data())) - // // doWork() - // msg.Ack() - // }() - } - - waitDone := make(chan struct{}) - go func() { - wg.Wait() - close(waitDone) - }() - - select { - case <-waitDone: - logger.Info().Msg("all workers drained") - case <-time.After(30 * time.Second): - logger.Info().Msg("drain timeout") - } - - logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) - nc.Flush() - nc.Close() -} From bb6da4966a0792dafcb40673908d72bf8660a586 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 26 Jun 2026 16:24:41 -0300 Subject: [PATCH 04/65] refactor main loop into app/runner.go --- .../subdomain-takeover/cmd/service/main.go | 50 ++++++++ .../subdomain-takeover/internal/app/runner.go | 89 +++++++++++++++ .../subdomain-takeover/internal/app/worker.go | 1 + .../{ => internal/config}/config.go | 4 +- .../detect/classifier.go} | 0 .../detect}/confidence.go | 0 .../types.go => internal/detect/evidence.go} | 0 .../detect/remediation.go} | 0 .../publisher.go => internal/detect/rules.go} | 0 .../logging/logger.go} | 0 .../messaging/client.go} | 0 .../internal/messaging/consumer.go | 0 .../internal/messaging/publisher.go | 0 .../internal/model/finding.go | 0 .../internal/model/types.go | 0 .../internal/scanner/confidence.go | 0 .../internal/scanner/dns_checks.go | 0 .../internal/scanner/scanner.go | 0 scanners/subdomain-takeover/main.go | 107 ------------------ 19 files changed, 142 insertions(+), 109 deletions(-) create mode 100644 scanners/subdomain-takeover/cmd/service/main.go create mode 100644 scanners/subdomain-takeover/internal/app/runner.go create mode 100644 scanners/subdomain-takeover/internal/app/worker.go rename scanners/subdomain-takeover/{ => internal/config}/config.go (95%) rename scanners/subdomain-takeover/{model/finding.go => internal/detect/classifier.go} (100%) rename scanners/subdomain-takeover/{scanner => internal/detect}/confidence.go (100%) rename scanners/subdomain-takeover/{model/types.go => internal/detect/evidence.go} (100%) rename scanners/subdomain-takeover/{nats/consumer.go => internal/detect/remediation.go} (100%) rename scanners/subdomain-takeover/{nats/publisher.go => internal/detect/rules.go} (100%) rename scanners/subdomain-takeover/{scanner/dns_checks.go => internal/logging/logger.go} (100%) rename scanners/subdomain-takeover/{scanner/scanner.go => internal/messaging/client.go} (100%) create mode 100644 scanners/subdomain-takeover/internal/messaging/consumer.go create mode 100644 scanners/subdomain-takeover/internal/messaging/publisher.go create mode 100644 scanners/subdomain-takeover/internal/model/finding.go create mode 100644 scanners/subdomain-takeover/internal/model/types.go create mode 100644 scanners/subdomain-takeover/internal/scanner/confidence.go create mode 100644 scanners/subdomain-takeover/internal/scanner/dns_checks.go create mode 100644 scanners/subdomain-takeover/internal/scanner/scanner.go delete mode 100644 scanners/subdomain-takeover/main.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go new file mode 100644 index 0000000000..4fafed7523 --- /dev/null +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "context" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/app" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +func main() { + cfg := config.InitConfig() + logger := cfg.Logger + + ctx, cancel := context.WithCancel(context.Background()) + + nc, _ := nats.Connect(cfg.NatsUrl) + js, _ := jetstream.New(nc) + logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) + + s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ + Name: cfg.NatsStream, + Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, + }) + + cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: cfg.DurableName, + AckPolicy: jetstream.AckExplicitPolicy, + }) + + iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) + if err != nil { + logger.Err(err).Msg("Error creating message iterator") + } + + app.Run( + app.RunnerDeps{ + Logger: logger, + WorkerCount: cfg.WorkerCount, + Iter: iter, + Cancel: cancel, + Context: ctx, + }) + + logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) + nc.Flush() + nc.Close() +} diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go new file mode 100644 index 0000000000..f02d48800d --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -0,0 +1,89 @@ +package app + +import ( + "context" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type RunnerDeps struct { + Logger zerolog.Logger + WorkerCount int + Iter jetstream.MessagesContext + Cancel context.CancelFunc + Context context.Context +} + +func Run(deps RunnerDeps) { + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + + var wg sync.WaitGroup + sem := make(chan struct{}, deps.WorkerCount) + + go func() { + <-sig + deps.Logger.Info().Msg("Shutdown requested...") + deps.Cancel() + deps.Iter.Stop() + }() + +Loop: + for { + // if nc.IsClosed() { + // deps.Logger.Error().Msg("Connection to NATS is closed.") + // break Loop + // } + + select { + case <-deps.Context.Done(): + break Loop + default: + } + + msg, err := deps.Iter.Next() + if err != nil { + if deps.Context.Err() != nil { + break Loop + } + deps.Logger.Debug().Err(err).Msg("next returned, continuing") + continue + } + + select { + case sem <- struct{}{}: + case <-deps.Context.Done(): + break Loop + } + + wg.Add(1) + go func(m jetstream.Msg) { + defer wg.Done() + defer func() { <-sem }() + deps.Logger.Info().Msg(string(m.Data())) + err := m.Ack() + if err != nil { + deps.Logger.Err(err).Msg("Ack error") + } + }(msg) + } + + waitDone := make(chan struct{}) + go func() { + wg.Wait() + close(waitDone) + }() + + select { + case <-waitDone: + deps.Logger.Info().Msg("all workers drained") + case <-time.After(30 * time.Second): + deps.Logger.Info().Msg("drain timeout") + } +} diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go new file mode 100644 index 0000000000..4879f7a483 --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -0,0 +1 @@ +package app diff --git a/scanners/subdomain-takeover/config.go b/scanners/subdomain-takeover/internal/config/config.go similarity index 95% rename from scanners/subdomain-takeover/config.go rename to scanners/subdomain-takeover/internal/config/config.go index b14c80710a..28d2ddf824 100644 --- a/scanners/subdomain-takeover/config.go +++ b/scanners/subdomain-takeover/internal/config/config.go @@ -1,4 +1,4 @@ -package main +package config import ( "os" @@ -18,7 +18,7 @@ type Config struct { Logger zerolog.Logger } -func initConfig() *Config { +func InitConfig() *Config { var cfg Config if err := envconfig.Process("", &cfg); err != nil { panic(err) diff --git a/scanners/subdomain-takeover/model/finding.go b/scanners/subdomain-takeover/internal/detect/classifier.go similarity index 100% rename from scanners/subdomain-takeover/model/finding.go rename to scanners/subdomain-takeover/internal/detect/classifier.go diff --git a/scanners/subdomain-takeover/scanner/confidence.go b/scanners/subdomain-takeover/internal/detect/confidence.go similarity index 100% rename from scanners/subdomain-takeover/scanner/confidence.go rename to scanners/subdomain-takeover/internal/detect/confidence.go diff --git a/scanners/subdomain-takeover/model/types.go b/scanners/subdomain-takeover/internal/detect/evidence.go similarity index 100% rename from scanners/subdomain-takeover/model/types.go rename to scanners/subdomain-takeover/internal/detect/evidence.go diff --git a/scanners/subdomain-takeover/nats/consumer.go b/scanners/subdomain-takeover/internal/detect/remediation.go similarity index 100% rename from scanners/subdomain-takeover/nats/consumer.go rename to scanners/subdomain-takeover/internal/detect/remediation.go diff --git a/scanners/subdomain-takeover/nats/publisher.go b/scanners/subdomain-takeover/internal/detect/rules.go similarity index 100% rename from scanners/subdomain-takeover/nats/publisher.go rename to scanners/subdomain-takeover/internal/detect/rules.go diff --git a/scanners/subdomain-takeover/scanner/dns_checks.go b/scanners/subdomain-takeover/internal/logging/logger.go similarity index 100% rename from scanners/subdomain-takeover/scanner/dns_checks.go rename to scanners/subdomain-takeover/internal/logging/logger.go diff --git a/scanners/subdomain-takeover/scanner/scanner.go b/scanners/subdomain-takeover/internal/messaging/client.go similarity index 100% rename from scanners/subdomain-takeover/scanner/scanner.go rename to scanners/subdomain-takeover/internal/messaging/client.go diff --git a/scanners/subdomain-takeover/internal/messaging/consumer.go b/scanners/subdomain-takeover/internal/messaging/consumer.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/model/types.go b/scanners/subdomain-takeover/internal/model/types.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/scanner/confidence.go b/scanners/subdomain-takeover/internal/scanner/confidence.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/scanner/dns_checks.go b/scanners/subdomain-takeover/internal/scanner/dns_checks.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/internal/scanner/scanner.go b/scanners/subdomain-takeover/internal/scanner/scanner.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/subdomain-takeover/main.go b/scanners/subdomain-takeover/main.go deleted file mode 100644 index 102e03b573..0000000000 --- a/scanners/subdomain-takeover/main.go +++ /dev/null @@ -1,107 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "os/signal" - "sync" - "syscall" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" -) - -func main() { - config := initConfig() - logger := config.Logger - - ctx, cancel := context.WithCancel(context.Background()) - - sig := make(chan os.Signal, 1) - signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) - - var wg sync.WaitGroup - sem := make(chan struct{}, config.WorkerCount) - - nc, _ := nats.Connect(config.NatsUrl) - js, _ := jetstream.New(nc) - logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) - - s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ - Name: config.NatsStream, - Subjects: []string{config.SubjectIn, config.SubjectOut}, - }) - - cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ - Durable: config.DurableName, - AckPolicy: jetstream.AckExplicitPolicy, - }) - - iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) - if err != nil { - logger.Err(err) - } - - go func() { - <-sig - logger.Info().Msg("Shutdown requested...") - cancel() - iter.Stop() - }() - -Loop: - for { - if nc.IsClosed() { - logger.Error().Msg("Connection to NATS is closed.") - break Loop - } - - select { - case <-ctx.Done(): - break Loop - default: - } - - msg, err := iter.Next() - if err != nil { - if ctx.Err() != nil { - break Loop - } - logger.Debug().Err(err).Msg("next returned, continuing") - continue - } - - select { - case sem <- struct{}{}: - case <-ctx.Done(): - break Loop - } - - wg.Add(1) - go func(m jetstream.Msg) { - defer wg.Done() - defer func() { <-sem }() - fmt.Println(string(m.Data())) - m.Ack() - }(msg) - } - - waitDone := make(chan struct{}) - go func() { - wg.Wait() - close(waitDone) - }() - - select { - case <-waitDone: - logger.Info().Msg("all workers drained") - case <-time.After(30 * time.Second): - logger.Info().Msg("drain timeout") - } - - logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) - nc.Flush() - nc.Close() -} From ba669c435966f36a0cf933a2ea020703e4f6b291 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Mon, 29 Jun 2026 18:00:18 -0300 Subject: [PATCH 05/65] init worker and publisher objects --- .../subdomain-takeover/cmd/service/main.go | 60 ++++++++++++++----- .../subdomain-takeover/internal/app/runner.go | 45 ++++++-------- .../subdomain-takeover/internal/app/worker.go | 50 ++++++++++++++++ .../internal/messaging/client.go | 0 .../internal/messaging/consumer.go | 1 + .../internal/messaging/publisher.go | 36 +++++++++++ .../internal/model/finding.go | 13 ++++ .../internal/model/input.go | 17 ++++++ .../internal/model/types.go | 0 .../internal/scanner/confidence.go | 0 .../internal/scanner/dns_checks.go | 0 .../internal/scanner/scanner.go | 0 12 files changed, 180 insertions(+), 42 deletions(-) delete mode 100644 scanners/subdomain-takeover/internal/messaging/client.go create mode 100644 scanners/subdomain-takeover/internal/model/input.go delete mode 100644 scanners/subdomain-takeover/internal/model/types.go delete mode 100644 scanners/subdomain-takeover/internal/scanner/confidence.go delete mode 100644 scanners/subdomain-takeover/internal/scanner/dns_checks.go delete mode 100644 scanners/subdomain-takeover/internal/scanner/scanner.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 4fafed7523..5275a170f2 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -2,47 +2,79 @@ package main import ( "context" + "os" + "os/signal" + "syscall" "time" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/app" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" ) +func checkErr(err error, log zerolog.Logger, msg string, exit bool) { + if err != nil { + log.Err(err).Msg(msg) + if exit { + os.Exit(1) + } + } + +} + func main() { cfg := config.InitConfig() logger := cfg.Logger ctx, cancel := context.WithCancel(context.Background()) - nc, _ := nats.Connect(cfg.NatsUrl) - js, _ := jetstream.New(nc) + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + + nc, err := nats.Connect(cfg.NatsUrl) + checkErr(err, logger, "", true) + + js, err := jetstream.New(nc) + checkErr(err, logger, "", true) + logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) - s, _ := js.CreateStream(ctx, jetstream.StreamConfig{ + s, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ Name: cfg.NatsStream, Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, }) + checkErr(err, logger, "", true) - cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + cons, err := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ Durable: cfg.DurableName, AckPolicy: jetstream.AckExplicitPolicy, }) + checkErr(err, logger, "", true) iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) - if err != nil { - logger.Err(err).Msg("Error creating message iterator") + checkErr(err, logger, "", true) + + pub := messaging.NewPublisher(logger, js, cfg.SubjectOut) + worker := app.NewWorker(logger, *pub) + + go func() { + <-sig + logger.Info().Msg("Shutdown requested...") + cancel() + iter.Stop() + }() + + deps := app.RunnerDeps{ + Logger: logger, + WorkerCount: cfg.WorkerCount, + Iter: iter, + Worker: *worker, } - app.Run( - app.RunnerDeps{ - Logger: logger, - WorkerCount: cfg.WorkerCount, - Iter: iter, - Cancel: cancel, - Context: ctx, - }) + app.Run(ctx, deps) logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) nc.Flush() diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index f02d48800d..a41650355d 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -2,10 +2,7 @@ package app import ( "context" - "os" - "os/signal" "sync" - "syscall" "time" "github.com/nats-io/nats.go/jetstream" @@ -16,49 +13,45 @@ type RunnerDeps struct { Logger zerolog.Logger WorkerCount int Iter jetstream.MessagesContext - Cancel context.CancelFunc - Context context.Context + Worker Worker } -func Run(deps RunnerDeps) { - sig := make(chan os.Signal, 1) - signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) +func Run(ctx context.Context, deps RunnerDeps) { + logger := deps.Logger + iter := deps.Iter + + if deps.WorkerCount < 1 { + deps.WorkerCount = 1 + } var wg sync.WaitGroup sem := make(chan struct{}, deps.WorkerCount) - go func() { - <-sig - deps.Logger.Info().Msg("Shutdown requested...") - deps.Cancel() - deps.Iter.Stop() - }() - Loop: for { // if nc.IsClosed() { - // deps.Logger.Error().Msg("Connection to NATS is closed.") + // logger.Error().Msg("Connection to NATS is closed.") // break Loop // } select { - case <-deps.Context.Done(): + case <-ctx.Done(): break Loop default: } - msg, err := deps.Iter.Next() + msg, err := iter.Next() if err != nil { - if deps.Context.Err() != nil { + if ctx.Err() != nil { break Loop } - deps.Logger.Debug().Err(err).Msg("next returned, continuing") + logger.Debug().Err(err).Msg("next returned, continuing") continue } select { case sem <- struct{}{}: - case <-deps.Context.Done(): + case <-ctx.Done(): break Loop } @@ -66,11 +59,7 @@ Loop: go func(m jetstream.Msg) { defer wg.Done() defer func() { <-sem }() - deps.Logger.Info().Msg(string(m.Data())) - err := m.Ack() - if err != nil { - deps.Logger.Err(err).Msg("Ack error") - } + deps.Worker.Handle(ctx, m) }(msg) } @@ -82,8 +71,8 @@ Loop: select { case <-waitDone: - deps.Logger.Info().Msg("all workers drained") + logger.Info().Msg("all workers drained") case <-time.After(30 * time.Second): - deps.Logger.Info().Msg("drain timeout") + logger.Info().Msg("drain timeout") } } diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index 4879f7a483..44e5452724 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -1 +1,51 @@ package app + +import ( + "bytes" + "context" + "encoding/json" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type Worker struct { + logger zerolog.Logger + pub messaging.Publisher +} + +func NewWorker(logger zerolog.Logger, pub messaging.Publisher) *Worker { + return &Worker{logger: logger, pub: pub} +} + +func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { + log := w.logger.With().Str("component", "worker").Logger() + // decode -> classify -> publish -> ack + scan, err := decodeScan(msg.Data()) + if err != nil { + log.Err(err).Msg("Decoding error") + } + log.Info().Msg(scan.Domain) + + // var finding model.Finding + // err := w.pub.Publish(ctx, finding) + // if err != nil { + // return err + // } + + if err := msg.Ack(); err != nil { + log.Err(err).Msg("Ack error") + return err + } + + return nil +} + +func decodeScan(data []byte) (model.Input, error) { + var scan model.Input + trimmed := bytes.Trim(data, "\n") + err := json.Unmarshal(trimmed, &scan) + return scan, err +} diff --git a/scanners/subdomain-takeover/internal/messaging/client.go b/scanners/subdomain-takeover/internal/messaging/client.go deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scanners/subdomain-takeover/internal/messaging/consumer.go b/scanners/subdomain-takeover/internal/messaging/consumer.go index e69de29bb2..cbb7a43f8b 100644 --- a/scanners/subdomain-takeover/internal/messaging/consumer.go +++ b/scanners/subdomain-takeover/internal/messaging/consumer.go @@ -0,0 +1 @@ +package messaging diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go index e69de29bb2..5b5585a75f 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -0,0 +1,36 @@ +package messaging + +import ( + "context" + "encoding/json" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type Publisher struct { + logger zerolog.Logger + js jetstream.JetStream + subject string +} + +func NewPublisher(logger zerolog.Logger, js jetstream.JetStream, subject string) *Publisher { + return &Publisher{logger: logger, js: js, subject: subject} +} + +func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { + payload, err := json.Marshal(finding) + if err != nil { + p.logger.Error().Err(err).Str("domain", finding.Domain).Msg("marshal finding failed") + return err + } + + if _, err := p.js.Publish(ctx, p.subject, payload); err != nil { + p.logger.Error().Err(err).Str("domain", finding.Domain).Str("subject", p.subject).Msg("publish failed") + return err + } + + p.logger.Debug().Str("domain", finding.Domain).Str("subject", p.subject).Msg("finding published") + return nil +} diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go index e69de29bb2..5d387c0f1d 100644 --- a/scanners/subdomain-takeover/internal/model/finding.go +++ b/scanners/subdomain-takeover/internal/model/finding.go @@ -0,0 +1,13 @@ +package model + +type Finding struct { + Domain string `json:"domain"` + DomainKey string `json:"domain_key"` + RecordType string `json:"record_type"` + Target string `json:"target"` + Provider string `json:"provider"` + LameType string `json:"lame_type"` + Confidence string `json:"confidence"` + ReasonCode string `json:"reason_code"` + Remediation string `json:"remediation"` +} diff --git a/scanners/subdomain-takeover/internal/model/input.go b/scanners/subdomain-takeover/internal/model/input.go new file mode 100644 index 0000000000..8d52ec9b96 --- /dev/null +++ b/scanners/subdomain-takeover/internal/model/input.go @@ -0,0 +1,17 @@ +package model + +type Input struct { + Domain string `json:"domain"` + BaseDomain string `json:"base_domain"` + ZoneApex string `json:"zone_apex"` + RecordExists bool `json:"record_exists"` + RCode string `json:"rcode"` + ResolveChain [][]string `json:"resolve_chain"` + CnameRecord string `json:"cname_record"` + NSRecords NSRecords `json:"ns_records"` +} + +type NSRecords struct { + Hostnames []string `json:"hostnames"` + Warnings []string `json:"warnings"` +} diff --git a/scanners/subdomain-takeover/internal/model/types.go b/scanners/subdomain-takeover/internal/model/types.go deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scanners/subdomain-takeover/internal/scanner/confidence.go b/scanners/subdomain-takeover/internal/scanner/confidence.go deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scanners/subdomain-takeover/internal/scanner/dns_checks.go b/scanners/subdomain-takeover/internal/scanner/dns_checks.go deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scanners/subdomain-takeover/internal/scanner/scanner.go b/scanners/subdomain-takeover/internal/scanner/scanner.go deleted file mode 100644 index e69de29bb2..0000000000 From b4e9b8cbbaed8c62e95d1dd6d0f6858fdb932ca9 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Mon, 29 Jun 2026 18:14:17 -0300 Subject: [PATCH 06/65] add health check at start of run loop --- .../subdomain-takeover/cmd/service/main.go | 1 + .../subdomain-takeover/internal/app/runner.go | 11 ++++++---- .../internal/messaging/health.go | 21 +++++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/messaging/health.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 5275a170f2..66085ffffa 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -72,6 +72,7 @@ func main() { WorkerCount: cfg.WorkerCount, Iter: iter, Worker: *worker, + NC: nc, } app.Run(ctx, deps) diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index a41650355d..42539405f0 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -5,6 +5,8 @@ import ( "sync" "time" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "github.com/rs/zerolog" ) @@ -14,6 +16,7 @@ type RunnerDeps struct { WorkerCount int Iter jetstream.MessagesContext Worker Worker + NC *nats.Conn } func Run(ctx context.Context, deps RunnerDeps) { @@ -29,10 +32,10 @@ func Run(ctx context.Context, deps RunnerDeps) { Loop: for { - // if nc.IsClosed() { - // logger.Error().Msg("Connection to NATS is closed.") - // break Loop - // } + if err := messaging.CheckConnection(deps.NC); err != nil { + logger.Error().Err(err).Msg("NATS connection unhealthy") + break Loop + } select { case <-ctx.Done(): diff --git a/scanners/subdomain-takeover/internal/messaging/health.go b/scanners/subdomain-takeover/internal/messaging/health.go new file mode 100644 index 0000000000..593ca1faff --- /dev/null +++ b/scanners/subdomain-takeover/internal/messaging/health.go @@ -0,0 +1,21 @@ +package messaging + +import ( + "errors" + "fmt" + + "github.com/nats-io/nats.go" +) + +func CheckConnection(nc *nats.Conn) error { + if nc == nil { + return errors.New("nats connection is nil") + } + if nc.IsClosed() { + return errors.New("nats connection is closed") + } + if !nc.IsConnected() { + return fmt.Errorf("nats not connected (status=%s)", nc.Status().String()) + } + return nil +} From 4afcd60c5b261f23506f60c6de112ca7b6261e10 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 3 Jul 2026 11:58:39 -0300 Subject: [PATCH 07/65] add init logic for cname takeover scanning --- .../subdomain-takeover/internal/app/worker.go | 3 + .../internal/detect/classifier.go | 1 + .../internal/detect/confidence.go | 1 + .../internal/detect/evidence.go | 31 +++ .../internal/detect/fingerprints.go | 247 ++++++++++++++++++ .../internal/detect/remediation.go | 1 + .../internal/detect/rules.go | 29 ++ .../internal/model/input.go | 8 +- 8 files changed, 317 insertions(+), 4 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprints.go diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index 44e5452724..7d000378e7 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" "github.com/nats-io/nats.go/jetstream" @@ -29,6 +30,8 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { } log.Info().Msg(scan.Domain) + detect.GetEvidence(scan) + // var finding model.Finding // err := w.pub.Publish(ctx, finding) // if err != nil { diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index e69de29bb2..ad6c03d18d 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -0,0 +1 @@ +package detect diff --git a/scanners/subdomain-takeover/internal/detect/confidence.go b/scanners/subdomain-takeover/internal/detect/confidence.go index e69de29bb2..ad6c03d18d 100644 --- a/scanners/subdomain-takeover/internal/detect/confidence.go +++ b/scanners/subdomain-takeover/internal/detect/confidence.go @@ -0,0 +1 @@ +package detect diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index e69de29bb2..c835b7444b 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -0,0 +1,31 @@ +package detect + +import ( + "strings" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +func GetEvidence(input model.Input) []model.Finding { + evidence := []model.Finding{} + + // parse CNAME chain + if input.CnameRecord != nil { + cname := parseCname(*input.CnameRecord) + cnameEvidence := GetCnameTakeoverEvidence(cname, input.Rcode) + evidence = append(evidence, *cnameEvidence) + } + + // parse NS hostnames + if len(input.NsRecords.Hostnames) > 0 { + nsEvidence := getNsTakeoverEvidence(input.NsRecords.Hostnames) + evidence = append(evidence, *nsEvidence) + } + + return evidence +} + +func parseCname(record string) string { + record_tokens := strings.Split(strings.Trim(record, "."), " ") + return record_tokens[len(record_tokens)-1] +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints.go b/scanners/subdomain-takeover/internal/detect/fingerprints.go new file mode 100644 index 0000000000..5d13a5489f --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprints.go @@ -0,0 +1,247 @@ +package detect + +import "strings" + +type Fingerprint struct { + Cname []string + Name string + Nxdomain bool + Fingerprint string +} + +func (f *Fingerprint) ContainsCname(target string) bool { + for _, cname := range f.Cname { + if strings.HasSuffix(target, cname) { + return true + } + } + return false +} + +var Fingerprints = []Fingerprint{ + { + Cname: []string{ + "elasticbeanstalk.com", + }, + Nxdomain: true, + Name: "AWS/Elastic Beanstalk", + Fingerprint: "NXDOMAIN", + }, + { + Cname: []string{ + "s3.amazonaws.com", + }, + Nxdomain: false, + Name: "AWS/S3", + Fingerprint: "The specified bucket does not exist", + }, + { + Cname: []string{ + "agilecrm.com", + }, + Nxdomain: false, + Name: "Agile CRM", + Fingerprint: "Sorry, this page is no longer available.", + }, + { + Cname: []string{ + "airee.ru", + }, + Nxdomain: false, + Name: "Airee.ru", + Fingerprint: "Ошибка 402. Сервис Айри.рф не оплачен", + }, + { + Cname: []string{ + "animaapp.io", + }, + Nxdomain: false, + Name: "Anima", + Fingerprint: "The page you were looking for does not exist.", + }, + { + Cname: []string{ + "bitbucket.io", + }, + Nxdomain: false, + Name: "Bitbucket", + Fingerprint: "Repository not found", + }, + { + Cname: []string{ + "trydiscourse.com", + }, + Nxdomain: true, + Name: "Discourse", + Fingerprint: "NXDOMAIN", + }, + { + Cname: []string{ + "furyns.com", + }, + Nxdomain: false, + Name: "Gemfury", + Fingerprint: "404: This page could not be found.", + }, + { + Cname: []string{ + "ghost.io", + }, + Nxdomain: false, + Name: "Ghost", + Fingerprint: "Site unavailable\\.|Failed to resolve DNS path for this host", + }, + { + Cname: []string{ + "hatenablog.com", + }, + Nxdomain: false, + Name: "HatenaBlog", + Fingerprint: "404 Blog is not found", + }, + { + Cname: []string{ + "helpjuice.com", + }, + Nxdomain: false, + Name: "Help Juice", + Fingerprint: "We could not find what you're looking for.", + }, + { + Cname: []string{ + "helpscoutdocs.com", + }, + Nxdomain: false, + Name: "Help Scout", + Fingerprint: "No settings were found for this company:", + }, + { + Cname: []string{ + "helprace.com", + }, + Nxdomain: false, + Name: "Helprace", + Fingerprint: "HTTP_STATUS=301", + }, + { + Cname: []string{ + "youtrack.cloud", + }, + Nxdomain: false, + Name: "JetBrains", + Fingerprint: "is not a registered InCloud YouTrack", + }, + { + Cname: []string{ + "launchrock.com", + }, + Nxdomain: false, + Name: "LaunchRock", + Fingerprint: "HTTP_STATUS=500", + }, + { + Cname: []string{ + "cloudapp.azure.com", + "azurewebsites.net", + "blob.core.windows.net", + "cloudapp.azure.com", + "azure-api.net", + "azurehdinsight.net", + "azureedge.net", + "azurecontainer.io", + "database.windows.net", + "azuredatalakestore.net", + "search.windows.net", + "azurecr.io", + "redis.cache.windows.net", + "azurehdinsight.net", + "servicebus.windows.net", + "trafficmanager.net", + "visualstudio.com", + }, + Nxdomain: true, + Name: "Microsoft Azure", + Fingerprint: "NXDOMAIN", + }, + { + Cname: []string{ + "ngrok.io", + }, + Nxdomain: false, + Name: "Ngrok", + Fingerprint: "Tunnel .*.ngrok.io not found", + }, + { + Cname: []string{ + "readme.io", + }, + Nxdomain: false, + Name: "Readme.io", + Fingerprint: "The creators of this project are still working on making everything perfect!", + }, + { + Cname: []string{ + "52.16.160.97", + }, + Nxdomain: false, + Name: "SmartJobBoard", + Fingerprint: "This job board website is either expired or its domain name is invalid.", + }, + { + Cname: []string{ + "s.strikinglydns.com", + }, + Nxdomain: false, + Name: "Strikingly", + Fingerprint: "PAGE NOT FOUND.", + }, + { + Cname: []string{ + "na-west1.surge.sh", + }, + Nxdomain: false, + Name: "Surge.sh", + Fingerprint: "project not found", + }, + { + Cname: []string{ + "surveysparrow.com", + }, + Nxdomain: false, + Name: "SurveySparrow", + Fingerprint: "Account not found.", + }, + { + Cname: []string{ + "read.uberflip.com", + }, + Nxdomain: false, + Name: "Uberflip", + Fingerprint: "The URL you've accessed does not provide a hub.", + }, + { + Cname: []string{ + "stats.uptimerobot.com", + }, + Nxdomain: false, + Name: "Uptimerobot", + Fingerprint: "page not found", + }, + { + Cname: []string{ + "wordpress.com", + }, + Nxdomain: false, + Name: "Wordpress", + Fingerprint: "Do you want to register .*.wordpress.com?", + }, + { + Cname: []string{ + "worksites.net", + "69.164.223.206", + }, + Nxdomain: false, + Name: "Worksites", + Fingerprint: "Hello! Sorry, but the website you’re looking for doesn’t exist.", + }, +} diff --git a/scanners/subdomain-takeover/internal/detect/remediation.go b/scanners/subdomain-takeover/internal/detect/remediation.go index e69de29bb2..ad6c03d18d 100644 --- a/scanners/subdomain-takeover/internal/detect/remediation.go +++ b/scanners/subdomain-takeover/internal/detect/remediation.go @@ -0,0 +1 @@ +package detect diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index e69de29bb2..ced758ce62 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -0,0 +1,29 @@ +package detect + +import ( + "fmt" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +func GetCnameTakeoverEvidence(cname string, rcode string) *model.Finding { + for _, fp := range Fingerprints { + if fp.ContainsCname(cname) { + fmt.Println(fp.Name) + if fp.Nxdomain { + fmt.Println("check for nxdomain on A record") + if rcode == "NXDOMAIN" { + fmt.Println("fingerprint match, CNAME takeover possible") + } + } else { + fmt.Println("check for other fingerprint") + } + } + } + + return nil +} + +func getNsTakeoverEvidence(hostnames []string) *model.Finding { + return nil +} diff --git a/scanners/subdomain-takeover/internal/model/input.go b/scanners/subdomain-takeover/internal/model/input.go index 8d52ec9b96..57a4da346a 100644 --- a/scanners/subdomain-takeover/internal/model/input.go +++ b/scanners/subdomain-takeover/internal/model/input.go @@ -5,13 +5,13 @@ type Input struct { BaseDomain string `json:"base_domain"` ZoneApex string `json:"zone_apex"` RecordExists bool `json:"record_exists"` - RCode string `json:"rcode"` + Rcode string `json:"rcode"` ResolveChain [][]string `json:"resolve_chain"` - CnameRecord string `json:"cname_record"` - NSRecords NSRecords `json:"ns_records"` + CnameRecord *string `json:"cname_record"` + NsRecords NsRecords `json:"ns_records"` } -type NSRecords struct { +type NsRecords struct { Hostnames []string `json:"hostnames"` Warnings []string `json:"warnings"` } From 26f6811ef2c6ac4532de022cfc041d54632ab72e Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Mon, 6 Jul 2026 15:51:16 -0300 Subject: [PATCH 08/65] return dict of rcode answers for each dns query type --- .../dns-scanner/dns_scanner/dns_scanner.py | 39 +++++++++++++++---- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index 573b986ea8..e1a720542f 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -17,7 +17,7 @@ TIMEOUT = int(os.getenv("SCAN_TIMEOUT", "20")) DNSSEC_NAMESERVER_IP = os.getenv("DNSSEC_NAMESERVER_IP") -DNSSEC_NAMESERVER_HOSTNAME= os.getenv("DNSSEC_NAMESERVER_HOSTNAME") +DNSSEC_NAMESERVER_HOSTNAME = os.getenv("DNSSEC_NAMESERVER_HOSTNAME") @dataclass @@ -27,6 +27,7 @@ class DNSScanResult: zone_apex: str = None record_exists: bool = None rcode: str = None + query_res: dict = None resolve_chain: list[list[str]] = None resolve_ips: [str] = None cname_record: str = None @@ -50,9 +51,9 @@ def find_zone_apex(domain, resolver=None): while name != dns.name.root: logger.debug(f"Checking for SOA at {name}") try: - answers = resolver.resolve(name, 'SOA') + answers = resolver.resolve(name, "SOA") logger.debug(f"Found SOA for {domain} at {name}: {answers[0]}") - zone_apex = str(name).rstrip('.') + zone_apex = str(name).rstrip(".") return zone_apex except NoAnswer: # Go up one level @@ -105,7 +106,13 @@ def minimal_dnssec_check(domain, nameserver_ip, nameserver_hostname, _ttl_marker """ try: q = dns.message.make_query(domain, dns.rdatatype.DNSKEY, want_dnssec=True) - resp = dns.query.tls(q, where=nameserver_ip, timeout=TIMEOUT, server_hostname=nameserver_hostname, verify=True) + resp = dns.query.tls( + q, + where=nameserver_ip, + timeout=TIMEOUT, + server_hostname=nameserver_hostname, + verify=True, + ) if resp.rcode() != dns.rcode.NOERROR: return None except Timeout: @@ -114,7 +121,9 @@ def minimal_dnssec_check(domain, nameserver_ip, nameserver_hostname, _ttl_marker ) return None except Exception as e: - logger.error(f"Error while running minimal DNSSEC check for domain '{domain}': {e}") + logger.error( + f"Error while running minimal DNSSEC check for domain '{domain}': {e}" + ) return None # Check if AD (Authenticated Data) flag is set, showing that the data is DNSSEC validated return bool(resp.flags & dns.flags.AD) @@ -201,8 +210,15 @@ def scan_domain(domain, dkim_selectors=None): # Check if domain exists dns_answer_return_types = [] - for query_type in [dns.rdatatype.A, dns.rdatatype.SOA, dns.rdatatype.NS]: + query_res = {} + for query_type in [ + dns.rdatatype.A, + dns.rdatatype.SOA, + dns.rdatatype.NS, + dns.rdatatype.CNAME, + ]: rtype = get_dns_return_type(domain, query_type) + query_res[dns.rdatatype.to_text(query_type)] = rtype if rtype == "NOERROR": dns_answer_return_types.append(rtype) break @@ -221,6 +237,7 @@ def scan_domain(domain, dkim_selectors=None): ) dns_answer_return_types.append(rtype) + scan_result.query_res = query_res if "NOERROR" not in dns_answer_return_types: if "SERVFAIL" in dns_answer_return_types: scan_result.rcode = "SERVFAIL" @@ -284,10 +301,16 @@ def scan_domain(domain, dkim_selectors=None): logger.debug(f"Skipping DNSSEC check for {domain} - No zone apex found") zone_dnssec_enabled = None elif not DNSSEC_NAMESERVER_IP or not DNSSEC_NAMESERVER_HOSTNAME: - logger.debug(f"Skipping DNSSEC check for {domain} - DNSSEC nameserver environment variables not set") + logger.debug( + f"Skipping DNSSEC check for {domain} - DNSSEC nameserver environment variables not set" + ) zone_dnssec_enabled = None else: - zone_dnssec_enabled = dnssec_check_with_ttl(domain=zone_apex, nameserver_ip=DNSSEC_NAMESERVER_IP, nameserver_hostname=DNSSEC_NAMESERVER_HOSTNAME) + zone_dnssec_enabled = dnssec_check_with_ttl( + domain=zone_apex, + nameserver_ip=DNSSEC_NAMESERVER_IP, + nameserver_hostname=DNSSEC_NAMESERVER_HOSTNAME, + ) scan_result.zone_dnssec_enabled = zone_dnssec_enabled From 314d207332ce402e452fbcca4b71ed004d5d6ff0 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 7 Jul 2026 17:01:38 -0300 Subject: [PATCH 09/65] add function to query and process ns delegations --- .../dns-scanner/dns_scanner/dns_scanner.py | 112 +++++++++++++++++- 1 file changed, 107 insertions(+), 5 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index e1a720542f..7441cf816f 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -28,6 +28,7 @@ class DNSScanResult: record_exists: bool = None rcode: str = None query_res: dict = None + ns_delegations: dict = None resolve_chain: list[list[str]] = None resolve_ips: [str] = None cname_record: str = None @@ -145,7 +146,7 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): result = {"wildcard_entry": False, "wildcard_sibling": False} try: wildcard_sibling_domain = re.sub(r"^[^.]+", "*", domain) - wildcard_record = dns.resolver.resolve( + wildcard_record = resolver.resolve( wildcard_sibling_domain, rdtype=dns.rdatatype.A, raise_on_no_answer=False, @@ -157,7 +158,7 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): try: # check for mail-only subdomain (e.g. mail.example.com) mx_records = resolver.resolve(qname=domain, rdtype=dns.rdatatype.MX) - wildcard_mx = dns.resolver.resolve( + wildcard_mx = resolver.resolve( wildcard_sibling_domain, rdtype=dns.rdatatype.MX, raise_on_no_answer=False, @@ -194,6 +195,104 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): return result +def dns_query_direct( + where: str, qname: str, qtype: str, recursion_desired: bool, timeout: int +): + query = dns.message.make_query(qname, dns.rdatatype.from_text(qtype), use_edns=True) + if not recursion_desired: + query.flags &= ~dns.flags.RD + return dns.query.udp(query, where=where, timeout=timeout) + + +def check_ns_delegations(domain, ns_records, resolver=None, timeout_sec=10): + ns_hosts = ns_records.get("hostnames", []) + # Always return structured output, even when empty + output = { + "ns_hosts": ns_hosts, + "ns_checks": [], # per-NS evidence rows + "ns_delegation": { + "total_ns": len(ns_hosts), + "authoritative_ok": 0, + "lame_count": 0, + "lame_type": "none", # none | partial | full + }, + } + + if len(ns_hosts) == 0: + output["ns_delegation"]["lame_type"] = "unknown" + return output + + if resolver is None: + resolver = dns.resolver.get_default_resolver() + + for host in ns_hosts: + row = { + "ns_host": host, + "qname": domain, + "qtype": "SOA", + "rcode": None, + "answered_authoritatively": False, + "error": None, + "timeout": False, + } + + try: + ns_ip = None + try: + ns_a = resolver.resolve(host, "A") + if ns_a: + ns_ip = ns_a[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + if ns_ip is None: + try: + ns_aaaa = resolver.resolve(host, "AAAA") + if ns_aaaa: + ns_ip = ns_aaaa[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + if ns_ip is None: + row["error"] = "ns_ip_resolution_failed" + output["ns_delegation"]["lame_count"] += 1 + output["ns_checks"].append(row) + continue + + res = dns_query_direct(ns_ip, domain, "SOA", False, timeout_sec) + row["rcode"] = dns.rcode.to_text(res.rcode()) + row["answered_authoritatively"] = bool(res.flags & dns.flags.AA) + + if row["answered_authoritatively"] and row["rcode"] in [ + "NOERROR", + "NXDOMAIN", + ]: + output["ns_delegation"]["authoritative_ok"] += 1 + else: + output["ns_delegation"]["lame_count"] += 1 + except Timeout: + row["timeout"] = True + row["error"] = "timeout" + output["ns_delegation"]["lame_count"] += 1 + except Exception as e: + row["error"] = str(e) + output["ns_delegation"]["lame_count"] += 1 + + output["ns_checks"].append(row) + + ok = output["ns_delegation"]["authoritative_ok"] + total = output["ns_delegation"]["total_ns"] + + if ok == total: + output["ns_delegation"]["lame_type"] = "none" + elif ok == 0: + output["ns_delegation"]["lame_type"] = "full" + else: + output["ns_delegation"]["lame_type"] = "partial" + + return output + + def scan_domain(domain, dkim_selectors=None): """ Scan a domain for DNS records @@ -221,10 +320,8 @@ def scan_domain(domain, dkim_selectors=None): query_res[dns.rdatatype.to_text(query_type)] = rtype if rtype == "NOERROR": dns_answer_return_types.append(rtype) - break elif rtype is None: dns_answer_return_types.append(None) - continue elif rtype == "NXDOMAIN": scan_result.rcode = rtype scan_result.record_exists = False @@ -320,12 +417,17 @@ def scan_domain(domain, dkim_selectors=None): dmarc_scanner = DMARCScanner(domain) dmarc_scan_result = dmarc_scanner.run() scan_result.base_domain = dmarc_scan_result.get("base_domain", "") - scan_result.ns_records = dmarc_scan_result.get("ns", {}) scan_result.mx_records = dmarc_scan_result.get("mx", {}) scan_result.spf = dmarc_scan_result.get("spf", {}) scan_result.dmarc = dmarc_scan_result.get("dmarc", {}) logger.debug(f"DMARC scan elapsed time: {time.monotonic() - dmarc_start_time}") + ns_records = dmarc_scan_result.get("ns", {"hostnames": [], "errors": []}) + scan_result.ns_records = ns_records + # check nameserver delegations + ns_delegations = check_ns_delegations(domain=domain, ns_records=ns_records) + scan_result.ns_delegations = ns_delegations + # If no MX records are found (with warnings), but there are CNAME records, check the CNAME target for MX records if ( len(scan_result.mx_records.get("hosts", [])) == 0 From ddefa536c64b32ef42d3661af333029583440786 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 7 Jul 2026 17:30:06 -0300 Subject: [PATCH 10/65] use zone apex for probe qname and recheck for ns hostnames --- .../dns-scanner/dns_scanner/dns_scanner.py | 63 ++++++++++++------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index 7441cf816f..f82113bdeb 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -204,8 +204,42 @@ def dns_query_direct( return dns.query.udp(query, where=where, timeout=timeout) -def check_ns_delegations(domain, ns_records, resolver=None, timeout_sec=10): +def get_ns_ip(host: str, resolver): + ns_ip = None + try: + ns_a = resolver.resolve(host, "A") + if ns_a: + ns_ip = ns_a[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + if ns_ip is None: + try: + ns_aaaa = resolver.resolve(host, "AAAA") + if ns_aaaa: + ns_ip = ns_aaaa[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + return ns_ip + + +def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_sec=10): + if resolver is None: + resolver = dns.resolver.get_default_resolver() + + qname = zone_apex + if not zone_apex: + qname = domain + ns_hosts = ns_records.get("hostnames", []) + if len(ns_records) == 0: + try: + ns_res = resolver.resolve(domain, dns.rdatatype.NS) + ns_hosts = [host.to_text() for host in ns_res] + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_hosts = [] + # Always return structured output, even when empty output = { "ns_hosts": ns_hosts, @@ -217,18 +251,14 @@ def check_ns_delegations(domain, ns_records, resolver=None, timeout_sec=10): "lame_type": "none", # none | partial | full }, } - if len(ns_hosts) == 0: output["ns_delegation"]["lame_type"] = "unknown" return output - if resolver is None: - resolver = dns.resolver.get_default_resolver() - for host in ns_hosts: row = { "ns_host": host, - "qname": domain, + "qname": qname, "qtype": "SOA", "rcode": None, "answered_authoritatively": False, @@ -237,22 +267,7 @@ def check_ns_delegations(domain, ns_records, resolver=None, timeout_sec=10): } try: - ns_ip = None - try: - ns_a = resolver.resolve(host, "A") - if ns_a: - ns_ip = ns_a[0].to_text() - except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): - ns_ip = None - - if ns_ip is None: - try: - ns_aaaa = resolver.resolve(host, "AAAA") - if ns_aaaa: - ns_ip = ns_aaaa[0].to_text() - except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): - ns_ip = None - + ns_ip = get_ns_ip(host, resolver) if ns_ip is None: row["error"] = "ns_ip_resolution_failed" output["ns_delegation"]["lame_count"] += 1 @@ -425,7 +440,9 @@ def scan_domain(domain, dkim_selectors=None): ns_records = dmarc_scan_result.get("ns", {"hostnames": [], "errors": []}) scan_result.ns_records = ns_records # check nameserver delegations - ns_delegations = check_ns_delegations(domain=domain, ns_records=ns_records) + ns_delegations = check_ns_delegations( + domain=domain, zone_apex=zone_apex, ns_records=ns_records + ) scan_result.ns_delegations = ns_delegations # If no MX records are found (with warnings), but there are CNAME records, check the CNAME target for MX records From 0155dd634646c35117267014aded10340f6510a9 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 7 Jul 2026 17:35:01 -0300 Subject: [PATCH 11/65] better func names --- scanners/dns-scanner/dns_scanner/dns_scanner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index f82113bdeb..da2e3aca1b 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -195,7 +195,7 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): return result -def dns_query_direct( +def probe_nameserver( where: str, qname: str, qtype: str, recursion_desired: bool, timeout: int ): query = dns.message.make_query(qname, dns.rdatatype.from_text(qtype), use_edns=True) @@ -274,7 +274,7 @@ def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_s output["ns_checks"].append(row) continue - res = dns_query_direct(ns_ip, domain, "SOA", False, timeout_sec) + res = probe_nameserver(ns_ip, domain, "SOA", False, timeout_sec) row["rcode"] = dns.rcode.to_text(res.rcode()) row["answered_authoritatively"] = bool(res.flags & dns.flags.AA) From 9101a702d5c23ace565088f7234d2c43e95da848 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 8 Jul 2026 15:23:43 -0300 Subject: [PATCH 12/65] add new input fields --- .../internal/detect/classifier.go | 20 + .../internal/detect/evidence.go | 48 ++- .../internal/detect/fingerprints.go | 345 ++++++++++++++++++ .../internal/detect/remediation.go | 4 + .../internal/detect/rules.go | 30 +- .../internal/model/input.go | 47 ++- 6 files changed, 452 insertions(+), 42 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index ad6c03d18d..5e4a157c0d 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -1 +1,21 @@ package detect + +import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + +func Classify(input model.Input) ([]model.Finding, error) { + evidence := []model.Finding{} + + // parse CNAME chain + if input.CnameRecord != nil { + cnameEvidence := ExtractCNAMEEvidence(input) + evidence = append(evidence, *cnameEvidence) + } + + // parse NS hostnames + if len(input.NsRecords.Hostnames) > 0 { + nsEvidence := ExtractNSEvidence(input) + evidence = append(evidence, *nsEvidence) + } + + return evidence, nil +} diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index c835b7444b..f0ce3424e0 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -1,31 +1,49 @@ package detect import ( + "fmt" "strings" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) -func GetEvidence(input model.Input) []model.Finding { - evidence := []model.Finding{} - - // parse CNAME chain - if input.CnameRecord != nil { - cname := parseCname(*input.CnameRecord) - cnameEvidence := GetCnameTakeoverEvidence(cname, input.Rcode) - evidence = append(evidence, *cnameEvidence) +func ExtractCNAMEEvidence(input model.Input) *model.Finding { + cname := parseCname(*input.CnameRecord) + for _, fp := range Fingerprints { + if fp.ContainsCname(cname) { + fmt.Println(fp.Name) + if fp.Nxdomain { + fmt.Println("check for nxdomain on A record") + if input.QueryAnswers.A == "NXDOMAIN" { + fmt.Println("fingerprint match, CNAME takeover possible") + return &model.Finding{ + Domain: input.Domain, + RecordType: "CNAME", + Provider: fp.Name, + Confidence: "confirmed", + Target: cname, + } + } + } else { + fmt.Println("check for other fingerprint") + } + } } - // parse NS hostnames - if len(input.NsRecords.Hostnames) > 0 { - nsEvidence := getNsTakeoverEvidence(input.NsRecords.Hostnames) - evidence = append(evidence, *nsEvidence) - } + return nil +} + +func ExtractNSEvidence(input model.Input) *model.Finding { + return nil +} - return evidence +func ClassifyLameType(nsChecks []model.NsCheck) { + return } func parseCname(record string) string { - record_tokens := strings.Split(strings.Trim(record, "."), " ") + trimmed := strings.Trim(record, ".") + lower := strings.ToLower(trimmed) + record_tokens := strings.Split(lower, " ") return record_tokens[len(record_tokens)-1] } diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints.go b/scanners/subdomain-takeover/internal/detect/fingerprints.go index 5d13a5489f..9c4e642e1b 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprints.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprints.go @@ -9,6 +9,351 @@ type Fingerprint struct { Fingerprint string } +type NSProviderStatus string + +const ( + NSStatusVulnerable NSProviderStatus = "vulnerable" + NSStatusNotVulnerable NSProviderStatus = "not_vulnerable" + NSStatusEdgeCase NSProviderStatus = "edge_case" + NSStatusVulnerableWithPurchase NSProviderStatus = "vulnerable_with_purchase" + NSStatusRegistrationClosed NSProviderStatus = "registration_closed" +) + +type NSProviderFingerprint struct { + Name string + ProviderURL string + Status NSProviderStatus + HostPatterns []string + InstructionsURL string + PrivateDNS bool +} + +func (f *NSProviderFingerprint) ContainsNSHost(host string) bool { + host = strings.ToLower(strings.TrimSuffix(host, ".")) + for _, pattern := range f.HostPatterns { + if wildcardHostMatch(pattern, host) { + return true + } + } + return false +} + +func wildcardHostMatch(pattern, host string) bool { + pattern = strings.ToLower(strings.TrimSuffix(pattern, ".")) + if !strings.Contains(pattern, "*") { + return host == pattern + } + + parts := strings.Split(pattern, "*") + if len(parts) == 2 { + return strings.HasPrefix(host, parts[0]) && strings.HasSuffix(host, parts[1]) + } + + idx := 0 + for i, part := range parts { + if part == "" { + continue + } + + pos := strings.Index(host[idx:], part) + if pos < 0 { + return false + } + + if i == 0 && !strings.HasPrefix(host, part) { + return false + } + + idx += pos + len(part) + } + + last := parts[len(parts)-1] + if last != "" && !strings.HasSuffix(host, last) { + return false + } + + return true +} + +var NSProviderFingerprints = []NSProviderFingerprint{ + { + Name: "000Domains", + ProviderURL: "https://000domains.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.000domains.com", "ns2.000domains.com", "fwns1.000domains.com", "fwns2.000domains.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/19", + }, + { + Name: "AWS Route 53", + ProviderURL: "https://aws.amazon.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns-*.awsdns-*.org", "ns-*.awsdns-*.co.uk", "ns-*.awsdns-*.com", "ns-*.awsdns-*.net"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/1", + }, + { + Name: "Azure (Microsoft)", + ProviderURL: "https://azure.microsoft.com/", + Status: NSStatusEdgeCase, + HostPatterns: []string{"ns1-*.azure-dns.com", "ns2-*.azure-dns.net", "ns3-*.azure-dns.org", "ns4-*.azure-dns.info"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/5", + }, + { + Name: "BigCommerce", + ProviderURL: "https://bigcommerce.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.bigcommerce.com", "ns2.bigcommerce.com", "ns3.bigcommerce.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/35", + }, + { + Name: "Bizland", + ProviderURL: "https://bizland.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.bizland.com", "ns2.bizland.com", "clickme.click2site.com", "clickme2.click2site.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/3", + }, + { + Name: "ClouDNS", + ProviderURL: "https://cloudns.net/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"*.cloudns.net"}, + }, + { + Name: "Cloudflare", + ProviderURL: "https://cloudflare.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"*.ns.cloudflare.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/10", + }, + { + Name: "Digital Ocean", + ProviderURL: "https://digitalocean.com/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.digitalocean.com", "ns2.digitalocean.com", "ns3.digitalocean.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/22", + }, + { + Name: "DNSMadeEasy", + ProviderURL: "https://dnsmadeeasy.com/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns*.dnsmadeeasy.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/6", + }, + { + Name: "DNSimple", + ProviderURL: "https://dnsimple.com/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.dnsimple.com", "ns2.dnsimple.com", "ns3.dnsimple.com", "ns4.dnsimple.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/16", + }, + { + Name: "Domain.com", + ProviderURL: "https://domain.com/", + Status: NSStatusVulnerableWithPurchase, + HostPatterns: []string{"ns1.domain.com", "ns2.domain.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/17", + }, + { + Name: "DomainPeople", + ProviderURL: "https://domainpeople.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.domainpeople.com", "ns2.domainpeople.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/14", + }, + { + Name: "Dotster", + ProviderURL: "https://dotster.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.dotster.com", "ns2.dotster.com", "ns1.nameresolve.com", "ns2.nameresolve.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/18", + }, + { + Name: "Dreamhost", + ProviderURL: "https://dreamhost.com/", + Status: NSStatusEdgeCase, + HostPatterns: []string{"ns1.dreamhost.com", "ns2.dreamhost.com", "ns3.dreamhost.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/40", + }, + { + Name: "EasyDNS", + ProviderURL: "https://easydns.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"dns1.easydns.com", "dns2.easydns.net", "dns3.easydns.org", "dns4.easydns.info"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/9", + }, + { + Name: "Gandi.net", + ProviderURL: "https://gandi.net/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"a.dns.gandi.net", "b.dns.gandi.net", "c.dns.gandi.net"}, + }, + { + Name: "Google Cloud", + ProviderURL: "https://cloud.google.com/", + Status: NSStatusEdgeCase, + HostPatterns: []string{"ns-cloud-*.googledomains.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/2", + }, + { + Name: "Hostinger (old NS)", + ProviderURL: "https://hostinger.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.hostinger.com", "ns2.hostinger.com"}, + }, + { + Name: "Hover", + ProviderURL: "https://hover.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.hover.com", "ns2.hover.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/21", + }, + { + Name: "Hurricane Electric", + ProviderURL: "https://dns.he.net/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.he.net", "ns2.he.net", "ns3.he.net", "ns4.he.net", "ns5.he.net"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/25", + }, + { + Name: "Linode", + ProviderURL: "https://linode.com/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.linode.com", "ns2.linode.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/26", + }, + { + Name: "MediaTemple (mt)", + ProviderURL: "https://mediatemple.net/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.mediatemple.net", "ns2.mediatemple.net"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/23", + }, + { + Name: "MyDomain", + ProviderURL: "https://mydomain.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns1.mydomain.com", "ns2.mydomain.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/4", + }, + { + Name: "Name.com", + ProviderURL: "https://name.com/", + Status: NSStatusVulnerableWithPurchase, + HostPatterns: []string{"ns1*.name.com", "ns2*.name.com", "ns3*.name.com", "ns4*.name.com"}, + InstructionsURL: "https://github.com/libertalialtd/can-i-take-over-dns/issues/8", + }, + { + Name: "Namecheap", + ProviderURL: "https://namecheap.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"*.namecheaphosting.com", "*.registrar-servers.com"}, + }, + { + Name: "Network Solutions", + ProviderURL: "https://networksolutions.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.worldnic.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/15", + }, + { + Name: "NS1", + ProviderURL: "https://nsone.net/", + Status: NSStatusRegistrationClosed, + HostPatterns: []string{"dns1.p*.nsone.net", "dns2.p*.nsone.net", "dns3.p*.nsone.net", "dns4.p*.nsone.net"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/7", + }, + { + Name: "TierraNet", + ProviderURL: "https://tierra.net/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.domaindiscover.com", "ns2.domaindiscover.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/24", + }, + { + Name: "Reg.ru", + ProviderURL: "https://reg.ru/", + Status: NSStatusVulnerable, + HostPatterns: []string{"ns1.reg.ru", "ns2.reg.ru"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/28", + }, + { + Name: "UltraDNS", + ProviderURL: "https://www.home.neustar/dns-services/ultra-dns", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"pdns*.ultradns.com", "udns*.ultradns.com", "sdns*.ultradns.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/29", + }, + { + Name: "Yahoo Small Business", + ProviderURL: "https://yahoosmallbusiness.com/", + Status: NSStatusVulnerableWithPurchase, + HostPatterns: []string{"yns1.yahoo.com", "yns2.yahoo.com"}, + InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/20", + }, + { + Name: "Activision", + ProviderURL: "https://activision.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.activision.com"}, + PrivateDNS: true, + }, + { + Name: "Adobe", + ProviderURL: "https://adobe.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"adobe-dns-0*.adobe.com"}, + PrivateDNS: true, + }, + { + Name: "Apple", + ProviderURL: "https://apple.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"a.ns.apple.com", "b.ns.apple.com", "c.ns.apple.com", "d.ns.apple.com"}, + PrivateDNS: true, + }, + { + Name: "Automattic", + ProviderURL: "https://automattic.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.automattic.com"}, + PrivateDNS: true, + }, + { + Name: "Capital One", + ProviderURL: "https://capitalone.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.capitalone.com"}, + PrivateDNS: true, + }, + { + Name: "Disney", + ProviderURL: "https://disney.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.twdcns.com", "ns*.twdcns.info", "ns*.twdcns.co.uk"}, + PrivateDNS: true, + }, + { + Name: "Google", + ProviderURL: "https://google.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns*.google.com"}, + PrivateDNS: true, + }, + { + Name: "Lowe's", + ProviderURL: "https://lowes.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"authns*.lowes.com"}, + PrivateDNS: true, + }, + { + Name: "T-Mobile", + ProviderURL: "https://tmobileus.com/", + Status: NSStatusNotVulnerable, + HostPatterns: []string{"ns10.tmobileus.com", "ns10.tmobileus.net"}, + PrivateDNS: true, + }, +} + func (f *Fingerprint) ContainsCname(target string) bool { for _, cname := range f.Cname { if strings.HasSuffix(target, cname) { diff --git a/scanners/subdomain-takeover/internal/detect/remediation.go b/scanners/subdomain-takeover/internal/detect/remediation.go index ad6c03d18d..9d577baa7f 100644 --- a/scanners/subdomain-takeover/internal/detect/remediation.go +++ b/scanners/subdomain-takeover/internal/detect/remediation.go @@ -1 +1,5 @@ package detect + +func RemediationForReason(reasonCode string, provider string, lameType string) { + return +} diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index ced758ce62..1d2bb6c10a 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -1,29 +1,21 @@ package detect import ( - "fmt" - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) -func GetCnameTakeoverEvidence(cname string, rcode string) *model.Finding { - for _, fp := range Fingerprints { - if fp.ContainsCname(cname) { - fmt.Println(fp.Name) - if fp.Nxdomain { - fmt.Println("check for nxdomain on A record") - if rcode == "NXDOMAIN" { - fmt.Println("fingerprint match, CNAME takeover possible") - } - } else { - fmt.Println("check for other fingerprint") - } - } - } +func MatchCNAMEFingerprints(evidence model.Finding, fingerprints []Fingerprint) { + return +} + +func MatchNSProviderRules(evidence model.Finding, providerRules []string) { + return +} - return nil +func ShouldEmitCNAME() { + return } -func getNsTakeoverEvidence(hostnames []string) *model.Finding { - return nil +func ShouldEmitNSHijack() { + return } diff --git a/scanners/subdomain-takeover/internal/model/input.go b/scanners/subdomain-takeover/internal/model/input.go index 57a4da346a..7bdd179f73 100644 --- a/scanners/subdomain-takeover/internal/model/input.go +++ b/scanners/subdomain-takeover/internal/model/input.go @@ -1,17 +1,48 @@ package model type Input struct { - Domain string `json:"domain"` - BaseDomain string `json:"base_domain"` - ZoneApex string `json:"zone_apex"` - RecordExists bool `json:"record_exists"` - Rcode string `json:"rcode"` - ResolveChain [][]string `json:"resolve_chain"` - CnameRecord *string `json:"cname_record"` - NsRecords NsRecords `json:"ns_records"` + Domain string `json:"domain"` + BaseDomain string `json:"base_domain"` + ZoneApex string `json:"zone_apex"` + RecordExists bool `json:"record_exists"` + Rcode string `json:"rcode"` + ResolveChain [][]string `json:"resolve_chain"` + CnameRecord *string `json:"cname_record"` + NsRecords NsRecords `json:"ns_records"` + QueryAnswers QueryAnswers `json:"query_res"` + NsDelegations NsDelegations `json:"ns_delegations"` } type NsRecords struct { Hostnames []string `json:"hostnames"` Warnings []string `json:"warnings"` } + +type QueryAnswers struct { + A string `json:"A"` + SOA string `json:"SOA"` + NS string `json:"NS"` + CNAME string `json:"CNAME"` +} + +type NsDelegations struct { + Hosts []string `json:"ns_hosts"` + Checks []NsCheck `json:"ns_checks"` + Delegation Delegation `json:"ns_delegation"` +} + +type NsCheck struct { + Host string `json:"ns_host"` + Qname string `json:"qname"` + Qtype string `json:"qtype"` + Rcode string `json:"rcode"` + AnsweredAuthoritatively bool `json:"answered_authoritatively"` + Error *string `json:"error"` + Timeout bool `json:"timeout"` +} +type Delegation struct { + TotalHosts int `json:"total_ns"` + OkCount int `json:"authoritative_ok"` + LameCount int `json:"lame_count"` + LameType string `json:"lame_type"` +} From c213301480bc39bc8d5706a1d9571519dcbbb8e5 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 8 Jul 2026 15:24:50 -0300 Subject: [PATCH 13/65] add registrar checks and move new logic to ns_registrar.py --- .../dns-scanner/dns_scanner/dns_scanner.py | 121 +---------- .../dns-scanner/dns_scanner/ns_registrar.py | 195 ++++++++++++++++++ 2 files changed, 203 insertions(+), 113 deletions(-) create mode 100644 scanners/dns-scanner/dns_scanner/ns_registrar.py diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index da2e3aca1b..06b72869d5 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -12,6 +12,7 @@ from dns.exception import Timeout from dns_scanner.email_scanners import DKIMScanner, DMARCScanner +from dns_scanner.ns_registrar import check_ns_delegations, get_registrar_context logger = logging.getLogger(__name__) @@ -29,6 +30,7 @@ class DNSScanResult: rcode: str = None query_res: dict = None ns_delegations: dict = None + registrar_context: dict = None resolve_chain: list[list[str]] = None resolve_ips: [str] = None cname_record: str = None @@ -195,119 +197,6 @@ def get_wildcard_status(domain: str, resolver: Resolver, a_records: Answer): return result -def probe_nameserver( - where: str, qname: str, qtype: str, recursion_desired: bool, timeout: int -): - query = dns.message.make_query(qname, dns.rdatatype.from_text(qtype), use_edns=True) - if not recursion_desired: - query.flags &= ~dns.flags.RD - return dns.query.udp(query, where=where, timeout=timeout) - - -def get_ns_ip(host: str, resolver): - ns_ip = None - try: - ns_a = resolver.resolve(host, "A") - if ns_a: - ns_ip = ns_a[0].to_text() - except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): - ns_ip = None - - if ns_ip is None: - try: - ns_aaaa = resolver.resolve(host, "AAAA") - if ns_aaaa: - ns_ip = ns_aaaa[0].to_text() - except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): - ns_ip = None - - return ns_ip - - -def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_sec=10): - if resolver is None: - resolver = dns.resolver.get_default_resolver() - - qname = zone_apex - if not zone_apex: - qname = domain - - ns_hosts = ns_records.get("hostnames", []) - if len(ns_records) == 0: - try: - ns_res = resolver.resolve(domain, dns.rdatatype.NS) - ns_hosts = [host.to_text() for host in ns_res] - except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): - ns_hosts = [] - - # Always return structured output, even when empty - output = { - "ns_hosts": ns_hosts, - "ns_checks": [], # per-NS evidence rows - "ns_delegation": { - "total_ns": len(ns_hosts), - "authoritative_ok": 0, - "lame_count": 0, - "lame_type": "none", # none | partial | full - }, - } - if len(ns_hosts) == 0: - output["ns_delegation"]["lame_type"] = "unknown" - return output - - for host in ns_hosts: - row = { - "ns_host": host, - "qname": qname, - "qtype": "SOA", - "rcode": None, - "answered_authoritatively": False, - "error": None, - "timeout": False, - } - - try: - ns_ip = get_ns_ip(host, resolver) - if ns_ip is None: - row["error"] = "ns_ip_resolution_failed" - output["ns_delegation"]["lame_count"] += 1 - output["ns_checks"].append(row) - continue - - res = probe_nameserver(ns_ip, domain, "SOA", False, timeout_sec) - row["rcode"] = dns.rcode.to_text(res.rcode()) - row["answered_authoritatively"] = bool(res.flags & dns.flags.AA) - - if row["answered_authoritatively"] and row["rcode"] in [ - "NOERROR", - "NXDOMAIN", - ]: - output["ns_delegation"]["authoritative_ok"] += 1 - else: - output["ns_delegation"]["lame_count"] += 1 - except Timeout: - row["timeout"] = True - row["error"] = "timeout" - output["ns_delegation"]["lame_count"] += 1 - except Exception as e: - row["error"] = str(e) - output["ns_delegation"]["lame_count"] += 1 - - output["ns_checks"].append(row) - - ok = output["ns_delegation"]["authoritative_ok"] - total = output["ns_delegation"]["total_ns"] - - if ok == total: - output["ns_delegation"]["lame_type"] = "none" - elif ok == 0: - output["ns_delegation"]["lame_type"] = "full" - else: - output["ns_delegation"]["lame_type"] = "partial" - - return output - - def scan_domain(domain, dkim_selectors=None): """ Scan a domain for DNS records @@ -445,6 +334,12 @@ def scan_domain(domain, dkim_selectors=None): ) scan_result.ns_delegations = ns_delegations + registrar_domain = scan_result.base_domain or zone_apex or domain + scan_result.registrar_context = get_registrar_context( + base_domain=registrar_domain, + ns_hosts=ns_delegations.get("ns_hosts", []), + ) + # If no MX records are found (with warnings), but there are CNAME records, check the CNAME target for MX records if ( len(scan_result.mx_records.get("hosts", [])) == 0 diff --git a/scanners/dns-scanner/dns_scanner/ns_registrar.py b/scanners/dns-scanner/dns_scanner/ns_registrar.py new file mode 100644 index 0000000000..9b48e7754f --- /dev/null +++ b/scanners/dns-scanner/dns_scanner/ns_registrar.py @@ -0,0 +1,195 @@ +import os + +import dns +import dns.resolver +import requests +from dns.exception import Timeout +from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers + +TIMEOUT = int(os.getenv("SCAN_TIMEOUT", "20")) + + +def probe_nameserver( + where: str, qname: str, qtype: str, recursion_desired: bool, timeout: int +): + query = dns.message.make_query( + qname, + dns.rdatatype.from_text(qtype), + use_edns=True, + recursion_desired=recursion_desired, + ) + return dns.query.udp(query, where=where, timeout=timeout) + + +def get_ns_ip(host: str, resolver): + ns_ip = None + try: + ns_a = resolver.resolve(host, "A") + if ns_a: + ns_ip = ns_a[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + if ns_ip is None: + try: + ns_aaaa = resolver.resolve(host, "AAAA") + if ns_aaaa: + ns_ip = ns_aaaa[0].to_text() + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_ip = None + + return ns_ip + + +def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_sec=10): + if resolver is None: + resolver = dns.resolver.get_default_resolver() + + qname = zone_apex + if not zone_apex: + qname = domain + + ns_hosts = ns_records.get("hostnames", []) + if len(ns_records) == 0: + try: + ns_res = resolver.resolve(domain, dns.rdatatype.NS) + ns_hosts = [host.to_text() for host in ns_res] + except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): + ns_hosts = [] + + output = { + "ns_hosts": ns_hosts, + "ns_checks": [], + "ns_delegation": { + "total_ns": len(ns_hosts), + "authoritative_ok": 0, + "lame_count": 0, + "lame_type": "none", + }, + } + if len(ns_hosts) == 0: + output["ns_delegation"]["lame_type"] = "unknown" + return output + + for host in ns_hosts: + row = { + "ns_host": host, + "qname": qname, + "qtype": "SOA", + "rcode": None, + "answered_authoritatively": False, + "error": None, + "timeout": False, + } + + try: + ns_ip = get_ns_ip(host, resolver) + if ns_ip is None: + row["error"] = "ns_ip_resolution_failed" + output["ns_delegation"]["lame_count"] += 1 + output["ns_checks"].append(row) + continue + + res = probe_nameserver(ns_ip, qname, "SOA", False, timeout_sec) + row["rcode"] = dns.rcode.to_text(res.rcode()) + row["answered_authoritatively"] = bool(res.flags & dns.flags.AA) + + if row["answered_authoritatively"] and row["rcode"] in [ + "NOERROR", + "NXDOMAIN", + ]: + output["ns_delegation"]["authoritative_ok"] += 1 + else: + output["ns_delegation"]["lame_count"] += 1 + except Timeout: + row["timeout"] = True + row["error"] = "timeout" + output["ns_delegation"]["lame_count"] += 1 + except Exception as e: + row["error"] = str(e) + output["ns_delegation"]["lame_count"] += 1 + + output["ns_checks"].append(row) + + ok = output["ns_delegation"]["authoritative_ok"] + total = output["ns_delegation"]["total_ns"] + + if ok == total: + output["ns_delegation"]["lame_type"] = "none" + elif ok == 0: + output["ns_delegation"]["lame_type"] = "full" + else: + output["ns_delegation"]["lame_type"] = "partial" + + return output + + +def get_registrar_context(base_domain, ns_hosts=None): + context = { + "base_domain": base_domain, + "lookup_success": False, + "rdap_url": None, + "registrar_name": None, + "registrar_id": None, + "rdap_nameservers": [], + "delegation_matches_rdap": None, + "error": None, + } + + if not base_domain: + context["error"] = "missing_base_domain" + return context + + rdap_url = f"https://rdap.org/domain/{base_domain}" + context["rdap_url"] = rdap_url + + try: + response = requests.get(rdap_url, timeout=TIMEOUT) + response.raise_for_status() + payload = response.json() + except Exception as e: + context["error"] = str(e) + return context + + context["lookup_success"] = True + + nameservers = payload.get("nameservers", []) + context["rdap_nameservers"] = [ + (ns.get("ldhName") or "").rstrip(".").lower() + for ns in nameservers + if ns.get("ldhName") + ] + + entities = payload.get("entities", []) + for entity in entities: + roles = [r.lower() for r in entity.get("roles", [])] + if "registrar" not in roles: + continue + + context["registrar_id"] = entity.get("handle") + + vcard = entity.get("vcardArray", []) + if isinstance(vcard, list) and len(vcard) == 2 and isinstance(vcard[1], list): + for entry in vcard[1]: + if not isinstance(entry, list) or len(entry) < 4: + continue + key = entry[0] + value = entry[3] + if key in {"fn", "org"} and value: + context["registrar_name"] = value + break + + if context["registrar_name"] is None: + public_ids = entity.get("publicIds", []) + if public_ids: + context["registrar_name"] = public_ids[0].get("identifier") + + break + + if ns_hosts is not None: + normalized_hosts = {h.rstrip(".").lower() for h in ns_hosts if h} + rdap_hosts = set(context["rdap_nameservers"]) + if rdap_hosts: + context["delegation_matches_rdap"] = normalized_hosts == rdap_hosts + + return context From 8442f3adda185fa1c842fb5eb51b952d5cc9eeaa Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 8 Jul 2026 18:23:44 -0300 Subject: [PATCH 14/65] establish Classify() workflow --- .../subdomain-takeover/internal/app/worker.go | 20 ++++--- .../internal/detect/classifier.go | 25 +++++---- .../internal/detect/evidence.go | 52 ++++++++++--------- .../internal/detect/fingerprints.go | 6 +-- .../internal/detect/rules.go | 44 +++++++++++++--- 5 files changed, 93 insertions(+), 54 deletions(-) diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index 7d000378e7..40176d6c88 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -28,15 +28,21 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { if err != nil { log.Err(err).Msg("Decoding error") } - log.Info().Msg(scan.Domain) + // log.Info().Msg(scan.Domain) - detect.GetEvidence(scan) + findings, err := detect.Classify(scan) + if err != nil { + log.Err(err).Msg("classify error") + return err + } - // var finding model.Finding - // err := w.pub.Publish(ctx, finding) - // if err != nil { - // return err - // } + for _, finding := range findings { + err = w.pub.Publish(ctx, finding) + if err != nil { + log.Err(err).Msg("publish error") + return err + } + } if err := msg.Ack(); err != nil { log.Err(err).Msg("Ack error") diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index 5e4a157c0d..f14a5b7f35 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -3,19 +3,22 @@ package detect import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" func Classify(input model.Input) ([]model.Finding, error) { - evidence := []model.Finding{} + findings := []model.Finding{} - // parse CNAME chain - if input.CnameRecord != nil { - cnameEvidence := ExtractCNAMEEvidence(input) - evidence = append(evidence, *cnameEvidence) + cnameEvidence := ExtractCNAMEEvidence(input) + cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints) + if ShouldEmitCNAME(*cnameEvidence, cnameHit) { + findings = append(findings, model.Finding{ + Domain: input.Domain, + }) } - // parse NS hostnames - if len(input.NsRecords.Hostnames) > 0 { - nsEvidence := ExtractNSEvidence(input) - evidence = append(evidence, *nsEvidence) + nsEvidence := ExtractNSEvidence(input) + nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) + if ShouldEmitNSHijack(*nsEvidence, nsHit) { + findings = append(findings, model.Finding{ + Domain: input.Domain, + }) } - - return evidence, nil + return findings, nil } diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index f0ce3424e0..fb40318cfc 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -1,40 +1,42 @@ package detect import ( - "fmt" "strings" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) -func ExtractCNAMEEvidence(input model.Input) *model.Finding { - cname := parseCname(*input.CnameRecord) - for _, fp := range Fingerprints { - if fp.ContainsCname(cname) { - fmt.Println(fp.Name) - if fp.Nxdomain { - fmt.Println("check for nxdomain on A record") - if input.QueryAnswers.A == "NXDOMAIN" { - fmt.Println("fingerprint match, CNAME takeover possible") - return &model.Finding{ - Domain: input.Domain, - RecordType: "CNAME", - Provider: fp.Name, - Confidence: "confirmed", - Target: cname, - } - } - } else { - fmt.Println("check for other fingerprint") - } - } +// inputs from evidence.go +type CNAMEEvidence struct { + Domain string + Target string + QueryAnswer model.QueryAnswers // or just needed fields +} + +type NSEvidence struct { + Domain string + NSHosts []string + NSDelegations model.NsDelegations + // Registrar model.RegistrarContext // if/when added +} + +func ExtractCNAMEEvidence(input model.Input) *CNAMEEvidence { + if input.CnameRecord == nil { + return nil } - return nil + return &CNAMEEvidence{ + Domain: input.Domain, + Target: parseCname(*input.CnameRecord), + QueryAnswer: input.QueryAnswers, + } } -func ExtractNSEvidence(input model.Input) *model.Finding { - return nil +func ExtractNSEvidence(input model.Input) *NSEvidence { + if len(input.NsDelegations.Hosts) == 0 { + return nil + } + return &NSEvidence{} } func ClassifyLameType(nsChecks []model.NsCheck) { diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints.go b/scanners/subdomain-takeover/internal/detect/fingerprints.go index 9c4e642e1b..b7098341a2 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprints.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprints.go @@ -2,7 +2,7 @@ package detect import "strings" -type Fingerprint struct { +type CNAMEProviderFingerprint struct { Cname []string Name string Nxdomain bool @@ -354,7 +354,7 @@ var NSProviderFingerprints = []NSProviderFingerprint{ }, } -func (f *Fingerprint) ContainsCname(target string) bool { +func (f *CNAMEProviderFingerprint) ContainsTarget(target string) bool { for _, cname := range f.Cname { if strings.HasSuffix(target, cname) { return true @@ -363,7 +363,7 @@ func (f *Fingerprint) ContainsCname(target string) bool { return false } -var Fingerprints = []Fingerprint{ +var CNAMEProviderFingerprints = []CNAMEProviderFingerprint{ { Cname: []string{ "elasticbeanstalk.com", diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index 1d2bb6c10a..15edb479d6 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -1,21 +1,49 @@ package detect import ( - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "fmt" ) -func MatchCNAMEFingerprints(evidence model.Finding, fingerprints []Fingerprint) { - return +type CNAMEHit struct { + Matched bool + Provider string + ReasonCode string + RuleID string + NeedsNX bool } -func MatchNSProviderRules(evidence model.Finding, providerRules []string) { - return +type NSHit struct { + Matched bool + Provider string + ReasonCode string + RuleID string + ProviderVulnerable bool + RegistrarMismatch bool } -func ShouldEmitCNAME() { - return +func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint) *CNAMEHit { + for _, fp := range fingerprints { + if fp.ContainsTarget(evidence.Target) { + if fp.Nxdomain { + fmt.Println("check for nxdomain on A record") + if evidence.QueryAnswer.A == "NXDOMAIN" { + fmt.Println("Bingo") + } + } // else { + // fmt.Println("check for other fingerprint") + // } + } + } } -func ShouldEmitNSHijack() { +func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { return } + +func ShouldEmitCNAME(evidence CNAMEEvidence, hit *CNAMEHit) bool { + return false +} + +func ShouldEmitNSHijack(evidence NSEvidence, hit *NSHit) bool { + return false +} From c4c1582a16b045ade48036f8a4d53a57df9c400a Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 10:08:51 -0300 Subject: [PATCH 15/65] match cname fingerprints and emit finding --- .../internal/detect/classifier.go | 14 ++++- .../internal/detect/rules.go | 62 ++++++++++++++++--- 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index f14a5b7f35..0caea61196 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -7,15 +7,25 @@ func Classify(input model.Input) ([]model.Finding, error) { cnameEvidence := ExtractCNAMEEvidence(input) cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints) - if ShouldEmitCNAME(*cnameEvidence, cnameHit) { + if ShouldEmitCNAME(cnameHit) { + // compute confidence + // compute reason/remediation findings = append(findings, model.Finding{ - Domain: input.Domain, + Domain: cnameEvidence.Domain, + RecordType: "CNAME", + Target: cnameEvidence.Target, + Provider: cnameHit.Provider, + ReasonCode: cnameHit.ReasonCode, + Confidence: "", + Remediation: "", }) } nsEvidence := ExtractNSEvidence(input) nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) if ShouldEmitNSHijack(*nsEvidence, nsHit) { + // compute confidence + // compute reason/remediation findings = append(findings, model.Finding{ Domain: input.Domain, }) diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index 15edb479d6..9cb5ac5738 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -2,13 +2,15 @@ package detect import ( "fmt" + "io" + "net/http" + "strings" ) type CNAMEHit struct { Matched bool Provider string ReasonCode string - RuleID string NeedsNX bool } @@ -16,7 +18,6 @@ type NSHit struct { Matched bool Provider string ReasonCode string - RuleID string ProviderVulnerable bool RegistrarMismatch bool } @@ -24,26 +25,67 @@ type NSHit struct { func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint) *CNAMEHit { for _, fp := range fingerprints { if fp.ContainsTarget(evidence.Target) { - if fp.Nxdomain { - fmt.Println("check for nxdomain on A record") + hit := CNAMEHit{ + Matched: false, + Provider: fp.Name, + ReasonCode: "", + NeedsNX: fp.Nxdomain, + } + if hit.NeedsNX { + hit.ReasonCode = "CNAME_TARGET_MATCH_MISSING_NXDOMAIN" if evidence.QueryAnswer.A == "NXDOMAIN" { - fmt.Println("Bingo") + hit.Matched = true + hit.ReasonCode = "CNAME_DANGLING_NXDOMAIN" } - } // else { - // fmt.Println("check for other fingerprint") - // } + } else { + hit.ReasonCode = "CNAME_TARGET_MATCH_MISSING_BODY_FINGERPRINT" + if urlBodyContainsFingerprint(evidence.Domain, fp.Fingerprint) { + hit.Matched = true + hit.ReasonCode = "CNAME_PROVIDER_FINGERPRINT_BODY_MATCH" + } + } + return &hit } } + return nil } func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { - return + return &NSHit{} } -func ShouldEmitCNAME(evidence CNAMEEvidence, hit *CNAMEHit) bool { +func ShouldEmitCNAME(hit *CNAMEHit) bool { + if hit == nil { + return false + } + if hit.Matched { + return true + } return false } func ShouldEmitNSHijack(evidence NSEvidence, hit *NSHit) bool { return false } + +func urlBodyContainsFingerprint(domain string, fp string) bool { + // curl domain and look for matching fp in body + url := fmt.Sprintf("http://%s", domain) + res, err := http.Get(url) + if err != nil { + // log error + return false + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + // log err + return false + } + + if strings.Contains(string(body), fp) { + return true + } + return false +} From 61b90a5c44884f12ae4be2566ebcf91035d820f4 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 12:39:40 -0300 Subject: [PATCH 16/65] revert changes to record gate --- scanners/dns-scanner/dns_scanner/dns_scanner.py | 16 ++++------------ scanners/dns-scanner/dns_scanner/ns_registrar.py | 5 +++-- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index 06b72869d5..40567ba136 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -28,7 +28,6 @@ class DNSScanResult: zone_apex: str = None record_exists: bool = None rcode: str = None - query_res: dict = None ns_delegations: dict = None registrar_context: dict = None resolve_chain: list[list[str]] = None @@ -213,19 +212,14 @@ def scan_domain(domain, dkim_selectors=None): # Check if domain exists dns_answer_return_types = [] - query_res = {} - for query_type in [ - dns.rdatatype.A, - dns.rdatatype.SOA, - dns.rdatatype.NS, - dns.rdatatype.CNAME, - ]: + for query_type in [dns.rdatatype.A, dns.rdatatype.SOA, dns.rdatatype.NS]: rtype = get_dns_return_type(domain, query_type) - query_res[dns.rdatatype.to_text(query_type)] = rtype if rtype == "NOERROR": dns_answer_return_types.append(rtype) + break elif rtype is None: dns_answer_return_types.append(None) + continue elif rtype == "NXDOMAIN": scan_result.rcode = rtype scan_result.record_exists = False @@ -238,7 +232,6 @@ def scan_domain(domain, dkim_selectors=None): ) dns_answer_return_types.append(rtype) - scan_result.query_res = query_res if "NOERROR" not in dns_answer_return_types: if "SERVFAIL" in dns_answer_return_types: scan_result.rcode = "SERVFAIL" @@ -329,10 +322,9 @@ def scan_domain(domain, dkim_selectors=None): ns_records = dmarc_scan_result.get("ns", {"hostnames": [], "errors": []}) scan_result.ns_records = ns_records # check nameserver delegations - ns_delegations = check_ns_delegations( + scan_result.ns_delegations = check_ns_delegations( domain=domain, zone_apex=zone_apex, ns_records=ns_records ) - scan_result.ns_delegations = ns_delegations registrar_domain = scan_result.base_domain or zone_apex or domain scan_result.registrar_context = get_registrar_context( diff --git a/scanners/dns-scanner/dns_scanner/ns_registrar.py b/scanners/dns-scanner/dns_scanner/ns_registrar.py index 9b48e7754f..4e60d31dc4 100644 --- a/scanners/dns-scanner/dns_scanner/ns_registrar.py +++ b/scanners/dns-scanner/dns_scanner/ns_registrar.py @@ -16,8 +16,9 @@ def probe_nameserver( qname, dns.rdatatype.from_text(qtype), use_edns=True, - recursion_desired=recursion_desired, ) + if not recursion_desired: + query.flags &= ~dns.flags.RD return dns.query.udp(query, where=where, timeout=timeout) @@ -50,7 +51,7 @@ def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_s qname = domain ns_hosts = ns_records.get("hostnames", []) - if len(ns_records) == 0: + if len(ns_hosts) == 0: try: ns_res = resolver.resolve(domain, dns.rdatatype.NS) ns_hosts = [host.to_text() for host in ns_res] From c09a79000e6540d8562eefeeea251ef79c318859 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:14:24 -0300 Subject: [PATCH 17/65] run go mod tidy --- scanners/subdomain-takeover/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/subdomain-takeover/go.mod b/scanners/subdomain-takeover/go.mod index f4bfb6b567..ca7b20f4ac 100644 --- a/scanners/subdomain-takeover/go.mod +++ b/scanners/subdomain-takeover/go.mod @@ -4,6 +4,7 @@ go 1.25.0 require ( github.com/kelseyhightower/envconfig v1.4.0 + github.com/nats-io/nats.go v1.52.0 github.com/rs/zerolog v1.35.1 ) @@ -11,7 +12,6 @@ require ( github.com/klauspost/compress v1.18.5 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/nats-io/nats.go v1.52.0 // indirect github.com/nats-io/nkeys v0.4.15 // indirect github.com/nats-io/nuid v1.0.1 // indirect golang.org/x/crypto v0.49.0 // indirect From 7e55e8828483d9bd9e19af66d1ba4f4bb6b87f36 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:16:14 -0300 Subject: [PATCH 18/65] move logger out of config.go --- .../internal/bootstrap/logger.go | 13 +++++++++++ .../internal/config/config.go | 22 +++++-------------- .../internal/logging/logger.go | 0 3 files changed, 18 insertions(+), 17 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/bootstrap/logger.go delete mode 100644 scanners/subdomain-takeover/internal/logging/logger.go diff --git a/scanners/subdomain-takeover/internal/bootstrap/logger.go b/scanners/subdomain-takeover/internal/bootstrap/logger.go new file mode 100644 index 0000000000..3721d23e40 --- /dev/null +++ b/scanners/subdomain-takeover/internal/bootstrap/logger.go @@ -0,0 +1,13 @@ +package bootstrap + +import ( + "os" + + "github.com/rs/zerolog" +) + +func NewLogger(level zerolog.Level) zerolog.Logger { + zerolog.SetGlobalLevel(level) + multi := zerolog.MultiLevelWriter(zerolog.ConsoleWriter{Out: os.Stderr}) + return zerolog.New(multi).With().Timestamp().Logger() +} diff --git a/scanners/subdomain-takeover/internal/config/config.go b/scanners/subdomain-takeover/internal/config/config.go index 28d2ddf824..1b18e68117 100644 --- a/scanners/subdomain-takeover/internal/config/config.go +++ b/scanners/subdomain-takeover/internal/config/config.go @@ -1,37 +1,25 @@ package config import ( - "os" - "github.com/kelseyhightower/envconfig" "github.com/rs/zerolog" ) type Config struct { - NatsUrl string `envconfig:"NATS_URL" default:"nats://localhost:4222"` - NatsStream string `envconfig:"NATS_STREAM" default:"SCANS"` + NATSURL string `envconfig:"NATS_URL" default:"nats://localhost:4222"` + NATSStream string `envconfig:"NATS_STREAM" default:"SCANS"` SubjectIn string `envconfig:"SUBJECT_IN" default:"scans.dns_scanner_results"` SubjectOut string `envconfig:"SUBJECT_OUT" default:"scans.findings.upsert"` DurableName string `envconfig:"DURABLE_NAME" default:"subdomain_takeover"` WorkerCount int `envconfig:"WORKER_COUNT" default:"5"` LogLevel zerolog.Level `envconfig:"LOG_LEVEL" default:"info"` - Logger zerolog.Logger } -func InitConfig() *Config { +func Load() (*Config, error) { var cfg Config if err := envconfig.Process("", &cfg); err != nil { - panic(err) + return nil, err } - zerolog.SetGlobalLevel(cfg.LogLevel) - - // Create multiple output steams for zerolog - multi := zerolog.MultiLevelWriter(zerolog.ConsoleWriter{Out: os.Stderr}) - - logger := zerolog.New(multi).With().Timestamp().Logger() - - cfg.Logger = logger - - return &cfg + return &cfg, nil } diff --git a/scanners/subdomain-takeover/internal/logging/logger.go b/scanners/subdomain-takeover/internal/logging/logger.go deleted file mode 100644 index e69de29bb2..0000000000 From 7af7e0f892161ba16f1bc6c75495742b671cada8 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:16:57 -0300 Subject: [PATCH 19/65] bootstrap nats and add input interfaces --- .../subdomain-takeover/internal/app/runner.go | 6 +- .../subdomain-takeover/internal/app/worker.go | 47 +++++++++++---- .../internal/bootstrap/nats.go | 59 +++++++++++++++++++ 3 files changed, 99 insertions(+), 13 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/bootstrap/nats.go diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index 42539405f0..6d7ee52e8d 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -15,10 +15,14 @@ type RunnerDeps struct { Logger zerolog.Logger WorkerCount int Iter jetstream.MessagesContext - Worker Worker + Worker MessageHandler NC *nats.Conn } +type MessageHandler interface { + Handle(ctx context.Context, msg jetstream.Msg) error +} + func Run(ctx context.Context, deps RunnerDeps) { logger := deps.Logger iter := deps.Iter diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index 40176d6c88..e282fab952 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -4,48 +4,59 @@ import ( "bytes" "context" "encoding/json" + "fmt" - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" "github.com/nats-io/nats.go/jetstream" "github.com/rs/zerolog" ) +type FindingPublisher interface { + Publish(ctx context.Context, finding model.Finding) error +} + +type ScanClassifier interface { + Classify(input model.Input) ([]model.Finding, error) +} + type Worker struct { - logger zerolog.Logger - pub messaging.Publisher + logger zerolog.Logger + publisher FindingPublisher + classifier ScanClassifier } -func NewWorker(logger zerolog.Logger, pub messaging.Publisher) *Worker { - return &Worker{logger: logger, pub: pub} +func NewWorker(logger zerolog.Logger, publisher FindingPublisher, classifier ScanClassifier) *Worker { + return &Worker{logger: logger, publisher: publisher, classifier: classifier} } func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { log := w.logger.With().Str("component", "worker").Logger() - // decode -> classify -> publish -> ack + scan, err := decodeScan(msg.Data()) if err != nil { - log.Err(err).Msg("Decoding error") + log.Err(err).Msg("decode error") + w.term(msg, log, err) + return err } - // log.Info().Msg(scan.Domain) - findings, err := detect.Classify(scan) + findings, err := w.classifier.Classify(scan) if err != nil { log.Err(err).Msg("classify error") + w.nak(msg, log, err) return err } for _, finding := range findings { - err = w.pub.Publish(ctx, finding) + err = w.publisher.Publish(ctx, finding) if err != nil { log.Err(err).Msg("publish error") + w.nak(msg, log, err) return err } } if err := msg.Ack(); err != nil { - log.Err(err).Msg("Ack error") + log.Err(err).Msg("ack error") return err } @@ -58,3 +69,15 @@ func decodeScan(data []byte) (model.Input, error) { err := json.Unmarshal(trimmed, &scan) return scan, err } + +func (w *Worker) nak(msg jetstream.Msg, log zerolog.Logger, originalErr error) { + if err := msg.Nak(); err != nil { + log.Error().Err(fmt.Errorf("original=%v nak=%w", originalErr, err)).Msg("failed to nak message") + } +} + +func (w *Worker) term(msg jetstream.Msg, log zerolog.Logger, originalErr error) { + if err := msg.Term(); err != nil { + log.Error().Err(fmt.Errorf("original=%v term=%w", originalErr, err)).Msg("failed to term message") + } +} diff --git a/scanners/subdomain-takeover/internal/bootstrap/nats.go b/scanners/subdomain-takeover/internal/bootstrap/nats.go new file mode 100644 index 0000000000..51edb6b060 --- /dev/null +++ b/scanners/subdomain-takeover/internal/bootstrap/nats.go @@ -0,0 +1,59 @@ +package bootstrap + +import ( + "context" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +type RuntimeDeps struct { + NC *nats.Conn + Iter jetstream.MessagesContext + JS jetstream.JetStream +} + +func NewRuntimeDeps(ctx context.Context, cfg *config.Config) (*RuntimeDeps, error) { + nc, err := nats.Connect(cfg.NATSURL) + if err != nil { + return nil, err + } + + js, err := jetstream.New(nc) + if err != nil { + nc.Close() + return nil, err + } + + stream, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ + Name: cfg.NATSStream, + Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, + }) + if err != nil { + nc.Close() + return nil, err + } + + cons, err := stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: cfg.DurableName, + AckPolicy: jetstream.AckExplicitPolicy, + }) + if err != nil { + nc.Close() + return nil, err + } + + iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) + if err != nil { + nc.Close() + return nil, err + } + + return &RuntimeDeps{ + NC: nc, + Iter: iter, + JS: js, + }, nil +} From 7e4cdd9ba87fa811e0332be912622ab8d8269f31 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:17:37 -0300 Subject: [PATCH 20/65] add better error handling to main.go during startup and shutdown --- .../subdomain-takeover/cmd/service/main.go | 76 +++++++++---------- 1 file changed, 35 insertions(+), 41 deletions(-) diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 66085ffffa..0f588ec30d 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -8,76 +8,70 @@ import ( "time" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/app" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/bootstrap" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" "github.com/rs/zerolog" ) -func checkErr(err error, log zerolog.Logger, msg string, exit bool) { +func main() { + logger := bootstrap.NewLogger(zerolog.InfoLevel) + + cfg, err := config.Load() if err != nil { - log.Err(err).Msg(msg) - if exit { - os.Exit(1) - } + logger.Fatal().Err(err).Msg("failed to load config") } -} + logger = bootstrap.NewLogger(cfg.LogLevel) -func main() { - cfg := config.InitConfig() - logger := cfg.Logger + if err := detect.LoadFingerprints(); err != nil { + logger.Fatal().Err(err).Msg("failed to load fingerprints") + } ctx, cancel := context.WithCancel(context.Background()) + defer cancel() sig := make(chan os.Signal, 1) signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + defer signal.Stop(sig) - nc, err := nats.Connect(cfg.NatsUrl) - checkErr(err, logger, "", true) - - js, err := jetstream.New(nc) - checkErr(err, logger, "", true) - - logger.Info().Msgf("Connected to NATS at %s...", nc.ConnectedUrl()) - - s, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ - Name: cfg.NatsStream, - Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, - }) - checkErr(err, logger, "", true) - - cons, err := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ - Durable: cfg.DurableName, - AckPolicy: jetstream.AckExplicitPolicy, - }) - checkErr(err, logger, "", true) + runtimeDeps, err := bootstrap.NewRuntimeDeps(ctx, cfg) + if err != nil { + logger.Fatal().Err(err).Msg("failed to initialize runtime dependencies") + } - iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) - checkErr(err, logger, "", true) + logger.Info().Msgf("Connected to NATS at %s", runtimeDeps.NC.ConnectedUrl()) - pub := messaging.NewPublisher(logger, js, cfg.SubjectOut) - worker := app.NewWorker(logger, *pub) + pub := messaging.NewPublisher(logger, runtimeDeps.JS, cfg.SubjectOut) + matcher := detect.NewHTTPBodyFingerprintMatcher(5 * time.Second) + classifier := detect.NewClassifier(matcher) + worker := app.NewWorker(logger, pub, classifier) go func() { <-sig logger.Info().Msg("Shutdown requested...") cancel() - iter.Stop() + runtimeDeps.Iter.Stop() }() deps := app.RunnerDeps{ Logger: logger, WorkerCount: cfg.WorkerCount, - Iter: iter, - Worker: *worker, - NC: nc, + Iter: runtimeDeps.Iter, + Worker: worker, + NC: runtimeDeps.NC, } app.Run(ctx, deps) - logger.Info().Msgf("Disconnecting from NATS at %s", nc.ConnectedUrl()) - nc.Flush() - nc.Close() + logger.Info().Msgf("Disconnecting from NATS at %s", runtimeDeps.NC.ConnectedUrl()) + if err := runtimeDeps.NC.Flush(); err != nil { + logger.Error().Err(err).Msg("failed to flush nats connection") + } + runtimeDeps.NC.Close() + + if err := ctx.Err(); err != nil && err != context.Canceled { + logger.Error().Err(err).Msg("service exited with context error") + } } From 1d19257e48a5e24e6863cb27b8b9723f08f7a172 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:18:22 -0300 Subject: [PATCH 21/65] move fingerprint data fixtures to own json files --- .../detect/data/cname_fingerprints.json | 227 ++++++++ .../internal/detect/data/ns_fingerprints.json | 416 +++++++++++++ .../internal/detect/fingerprint_loader.go | 107 ++++ .../internal/detect/fingerprints.go | 546 +----------------- 4 files changed, 781 insertions(+), 515 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json create mode 100644 scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprint_loader.go diff --git a/scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json b/scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json new file mode 100644 index 0000000000..cbf3aa24be --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json @@ -0,0 +1,227 @@ +[ + { + "cname": [ + "elasticbeanstalk.com" + ], + "name": "AWS/Elastic Beanstalk", + "nxdomain": true, + "fingerprint": "NXDOMAIN" + }, + { + "cname": [ + "s3.amazonaws.com" + ], + "name": "AWS/S3", + "nxdomain": false, + "fingerprint": "The specified bucket does not exist" + }, + { + "cname": [ + "agilecrm.com" + ], + "name": "Agile CRM", + "nxdomain": false, + "fingerprint": "Sorry, this page is no longer available." + }, + { + "cname": [ + "airee.ru" + ], + "name": "Airee.ru", + "nxdomain": false, + "fingerprint": "Ошибка 402. Сервис Айри.рф не оплачен" + }, + { + "cname": [ + "animaapp.io" + ], + "name": "Anima", + "nxdomain": false, + "fingerprint": "The page you were looking for does not exist." + }, + { + "cname": [ + "bitbucket.io" + ], + "name": "Bitbucket", + "nxdomain": false, + "fingerprint": "Repository not found" + }, + { + "cname": [ + "trydiscourse.com" + ], + "name": "Discourse", + "nxdomain": true, + "fingerprint": "NXDOMAIN" + }, + { + "cname": [ + "furyns.com" + ], + "name": "Gemfury", + "nxdomain": false, + "fingerprint": "404: This page could not be found." + }, + { + "cname": [ + "ghost.io" + ], + "name": "Ghost", + "nxdomain": false, + "fingerprint": "Site unavailable\\.\u0026#124;Failed to resolve DNS path for this host" + }, + { + "cname": [ + "hatenablog.com" + ], + "name": "HatenaBlog", + "nxdomain": false, + "fingerprint": "404 Blog is not found" + }, + { + "cname": [ + "helpjuice.com" + ], + "name": "Help Juice", + "nxdomain": false, + "fingerprint": "We could not find what you're looking for." + }, + { + "cname": [ + "helpscoutdocs.com" + ], + "name": "Help Scout", + "nxdomain": false, + "fingerprint": "No settings were found for this company:" + }, + { + "cname": [ + "helprace.com" + ], + "name": "Helprace", + "nxdomain": false, + "fingerprint": "HTTP_STATUS=301" + }, + { + "cname": [ + "youtrack.cloud" + ], + "name": "JetBrains", + "nxdomain": false, + "fingerprint": "is not a registered InCloud YouTrack" + }, + { + "cname": [ + "launchrock.com" + ], + "name": "LaunchRock", + "nxdomain": false, + "fingerprint": "HTTP_STATUS=500" + }, + { + "cname": [ + "cloudapp.azure.com", + "azurewebsites.net", + "blob.core.windows.net", + "cloudapp.azure.com", + "azure-api.net", + "azurehdinsight.net", + "azureedge.net", + "azurecontainer.io", + "database.windows.net", + "azuredatalakestore.net", + "search.windows.net", + "azurecr.io", + "redis.cache.windows.net", + "azurehdinsight.net", + "servicebus.windows.net", + "trafficmanager.net", + "visualstudio.com" + ], + "name": "Microsoft Azure", + "nxdomain": true, + "fingerprint": "NXDOMAIN" + }, + { + "cname": [ + "ngrok.io" + ], + "name": "Ngrok", + "nxdomain": false, + "fingerprint": "Tunnel .*.ngrok.io not found" + }, + { + "cname": [ + "readme.io" + ], + "name": "Readme.io", + "nxdomain": false, + "fingerprint": "The creators of this project are still working on making everything perfect!" + }, + { + "cname": [ + "52.16.160.97" + ], + "name": "SmartJobBoard", + "nxdomain": false, + "fingerprint": "This job board website is either expired or its domain name is invalid." + }, + { + "cname": [ + "s.strikinglydns.com" + ], + "name": "Strikingly", + "nxdomain": false, + "fingerprint": "PAGE NOT FOUND." + }, + { + "cname": [ + "na-west1.surge.sh" + ], + "name": "Surge.sh", + "nxdomain": false, + "fingerprint": "project not found" + }, + { + "cname": [ + "surveysparrow.com" + ], + "name": "SurveySparrow", + "nxdomain": false, + "fingerprint": "Account not found." + }, + { + "cname": [ + "read.uberflip.com" + ], + "name": "Uberflip", + "nxdomain": false, + "fingerprint": "The URL you've accessed does not provide a hub." + }, + { + "cname": [ + "stats.uptimerobot.com" + ], + "name": "Uptimerobot", + "nxdomain": false, + "fingerprint": "page not found" + }, + { + "cname": [ + "wordpress.com" + ], + "name": "Wordpress", + "nxdomain": false, + "fingerprint": "Do you want to register .*.wordpress.com?" + }, + { + "cname": [ + "worksites.net", + "69.164.223.206" + ], + "name": "Worksites", + "nxdomain": false, + "fingerprint": "Hello! Sorry, but the website you\u0026rsquo;re looking for doesn\u0026rsquo;t exist." + } +] diff --git a/scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json b/scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json new file mode 100644 index 0000000000..85fd5262bf --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json @@ -0,0 +1,416 @@ +[ + { + "name": "000Domains", + "provider_url": "https://000domains.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.000domains.com", + "ns2.000domains.com", + "fwns1.000domains.com", + "fwns2.000domains.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/19" + }, + { + "name": "AWS Route 53", + "provider_url": "https://aws.amazon.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns-*.awsdns-*.org", + "ns-*.awsdns-*.co.uk", + "ns-*.awsdns-*.com", + "ns-*.awsdns-*.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/1" + }, + { + "name": "Azure (Microsoft)", + "provider_url": "https://azure.microsoft.com/", + "status": "edge_case", + "host_patterns": [ + "ns1-*.azure-dns.com", + "ns2-*.azure-dns.net", + "ns3-*.azure-dns.org", + "ns4-*.azure-dns.info" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/5" + }, + { + "name": "BigCommerce", + "provider_url": "https://bigcommerce.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.bigcommerce.com", + "ns2.bigcommerce.com", + "ns3.bigcommerce.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/35" + }, + { + "name": "Bizland", + "provider_url": "https://bizland.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.bizland.com", + "ns2.bizland.com", + "clickme.click2site.com", + "clickme2.click2site.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/3" + }, + { + "name": "ClouDNS", + "provider_url": "https://cloudns.net/", + "status": "not_vulnerable", + "host_patterns": [ + "*.cloudns.net" + ] + }, + { + "name": "Cloudflare", + "provider_url": "https://cloudflare.com/", + "status": "not_vulnerable", + "host_patterns": [ + "*.ns.cloudflare.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/10" + }, + { + "name": "Digital Ocean", + "provider_url": "https://digitalocean.com/", + "status": "vulnerable", + "host_patterns": [ + "ns1.digitalocean.com", + "ns2.digitalocean.com", + "ns3.digitalocean.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/22" + }, + { + "name": "DNSMadeEasy", + "provider_url": "https://dnsmadeeasy.com/", + "status": "vulnerable", + "host_patterns": [ + "ns*.dnsmadeeasy.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/6" + }, + { + "name": "DNSimple", + "provider_url": "https://dnsimple.com/", + "status": "vulnerable", + "host_patterns": [ + "ns1.dnsimple.com", + "ns2.dnsimple.com", + "ns3.dnsimple.com", + "ns4.dnsimple.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/16" + }, + { + "name": "Domain.com", + "provider_url": "https://domain.com/", + "status": "vulnerable_with_purchase", + "host_patterns": [ + "ns1.domain.com", + "ns2.domain.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/17" + }, + { + "name": "DomainPeople", + "provider_url": "https://domainpeople.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.domainpeople.com", + "ns2.domainpeople.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/14" + }, + { + "name": "Dotster", + "provider_url": "https://dotster.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.dotster.com", + "ns2.dotster.com", + "ns1.nameresolve.com", + "ns2.nameresolve.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/18" + }, + { + "name": "Dreamhost", + "provider_url": "https://dreamhost.com/", + "status": "edge_case", + "host_patterns": [ + "ns1.dreamhost.com", + "ns2.dreamhost.com", + "ns3.dreamhost.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/40" + }, + { + "name": "EasyDNS", + "provider_url": "https://easydns.com/", + "status": "not_vulnerable", + "host_patterns": [ + "dns1.easydns.com", + "dns2.easydns.net", + "dns3.easydns.org", + "dns4.easydns.info" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/9" + }, + { + "name": "Gandi.net", + "provider_url": "https://gandi.net/", + "status": "not_vulnerable", + "host_patterns": [ + "a.dns.gandi.net", + "b.dns.gandi.net", + "c.dns.gandi.net" + ] + }, + { + "name": "Google Cloud", + "provider_url": "https://cloud.google.com/", + "status": "edge_case", + "host_patterns": [ + "ns-cloud-*.googledomains.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/2" + }, + { + "name": "Hostinger (old NS)", + "provider_url": "https://hostinger.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.hostinger.com", + "ns2.hostinger.com" + ] + }, + { + "name": "Hover", + "provider_url": "https://hover.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.hover.com", + "ns2.hover.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/21" + }, + { + "name": "Hurricane Electric", + "provider_url": "https://dns.he.net/", + "status": "vulnerable", + "host_patterns": [ + "ns1.he.net", + "ns2.he.net", + "ns3.he.net", + "ns4.he.net", + "ns5.he.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/25" + }, + { + "name": "Linode", + "provider_url": "https://linode.com/", + "status": "vulnerable", + "host_patterns": [ + "ns1.linode.com", + "ns2.linode.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/26" + }, + { + "name": "MediaTemple (mt)", + "provider_url": "https://mediatemple.net/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.mediatemple.net", + "ns2.mediatemple.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/23" + }, + { + "name": "MyDomain", + "provider_url": "https://mydomain.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.mydomain.com", + "ns2.mydomain.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/4" + }, + { + "name": "Name.com", + "provider_url": "https://name.com/", + "status": "vulnerable_with_purchase", + "host_patterns": [ + "ns1*.name.com", + "ns2*.name.com", + "ns3*.name.com", + "ns4*.name.com" + ], + "instructions_url": "https://github.com/libertalialtd/can-i-take-over-dns/issues/8" + }, + { + "name": "Namecheap", + "provider_url": "https://namecheap.com/", + "status": "not_vulnerable", + "host_patterns": [ + "*.namecheaphosting.com", + "*.registrar-servers.com" + ] + }, + { + "name": "Network Solutions", + "provider_url": "https://networksolutions.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.worldnic.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/15" + }, + { + "name": "NS1", + "provider_url": "https://nsone.net/", + "status": "registration_closed", + "host_patterns": [ + "dns1.p*.nsone.net", + "dns2.p*.nsone.net", + "dns3.p*.nsone.net", + "dns4.p*.nsone.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/7" + }, + { + "name": "TierraNet", + "provider_url": "https://tierra.net/", + "status": "vulnerable", + "host_patterns": [ + "ns1.domaindiscover.com", + "ns2.domaindiscover.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/24" + }, + { + "name": "Reg.ru", + "provider_url": "https://reg.ru/", + "status": "vulnerable", + "host_patterns": [ + "ns1.reg.ru", + "ns2.reg.ru" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/28" + }, + { + "name": "UltraDNS", + "provider_url": "https://www.home.neustar/dns-services/ultra-dns", + "status": "not_vulnerable", + "host_patterns": [ + "pdns*.ultradns.com", + "udns*.ultradns.com", + "sdns*.ultradns.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/29" + }, + { + "name": "Yahoo Small Business", + "provider_url": "https://yahoosmallbusiness.com/", + "status": "vulnerable_with_purchase", + "host_patterns": [ + "yns1.yahoo.com", + "yns2.yahoo.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/20" + }, + { + "name": "Activision", + "provider_url": "https://activision.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.activision.com" + ], + "private_dns": true + }, + { + "name": "Adobe", + "provider_url": "https://adobe.com/", + "status": "not_vulnerable", + "host_patterns": [ + "adobe-dns-0*.adobe.com" + ], + "private_dns": true + }, + { + "name": "Apple", + "provider_url": "https://apple.com/", + "status": "not_vulnerable", + "host_patterns": [ + "a.ns.apple.com", + "b.ns.apple.com", + "c.ns.apple.com", + "d.ns.apple.com" + ], + "private_dns": true + }, + { + "name": "Automattic", + "provider_url": "https://automattic.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.automattic.com" + ], + "private_dns": true + }, + { + "name": "Capital One", + "provider_url": "https://capitalone.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.capitalone.com" + ], + "private_dns": true + }, + { + "name": "Disney", + "provider_url": "https://disney.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.twdcns.com", + "ns*.twdcns.info", + "ns*.twdcns.co.uk" + ], + "private_dns": true + }, + { + "name": "Google", + "provider_url": "https://google.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.google.com" + ], + "private_dns": true + }, + { + "name": "Lowe's", + "provider_url": "https://lowes.com/", + "status": "not_vulnerable", + "host_patterns": [ + "authns*.lowes.com" + ], + "private_dns": true + }, + { + "name": "T-Mobile", + "provider_url": "https://tmobileus.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns10.tmobileus.com", + "ns10.tmobileus.net" + ], + "private_dns": true + } +] diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go b/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go new file mode 100644 index 0000000000..694bf833aa --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go @@ -0,0 +1,107 @@ +package detect + +import ( + "embed" + "encoding/json" + "fmt" + "strings" + "sync" +) + +//go:embed data/*.json +var fingerprintFS embed.FS + +var ( + loadFingerprintsOnce sync.Once + loadFingerprintsErr error +) + +func LoadFingerprints() error { + loadFingerprintsOnce.Do(func() { + var cname []CNAMEProviderFingerprint + var ns []NSProviderFingerprint + + if err := loadJSON("data/cname_fingerprints.json", &cname); err != nil { + loadFingerprintsErr = err + return + } + + if err := loadJSON("data/ns_fingerprints.json", &ns); err != nil { + loadFingerprintsErr = err + return + } + + if err := validateCNAMEFingerprints(cname); err != nil { + loadFingerprintsErr = err + return + } + + if err := validateNSFingerprints(ns); err != nil { + loadFingerprintsErr = err + return + } + + CNAMEProviderFingerprints = cname + NSProviderFingerprints = ns + }) + + return loadFingerprintsErr +} + +func loadJSON(path string, out any) error { + b, err := fingerprintFS.ReadFile(path) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + + if err := json.Unmarshal(b, out); err != nil { + return fmt.Errorf("decode %s: %w", path, err) + } + + return nil +} + +func validateCNAMEFingerprints(fingerprints []CNAMEProviderFingerprint) error { + for i, fp := range fingerprints { + if strings.TrimSpace(fp.Name) == "" { + return fmt.Errorf("cname fingerprint[%d] missing name", i) + } + if len(fp.Cname) == 0 { + return fmt.Errorf("cname fingerprint[%d] has no cname patterns", i) + } + + for j, cname := range fp.Cname { + if strings.TrimSpace(cname) == "" { + return fmt.Errorf("cname fingerprint[%d] has empty cname pattern at index %d", i, j) + } + } + + if strings.TrimSpace(fp.Fingerprint) == "" { + return fmt.Errorf("cname fingerprint[%d] missing fingerprint", i) + } + + fingerprints[i].Mode = normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + } + + return nil +} + +func validateNSFingerprints(fingerprints []NSProviderFingerprint) error { + for i, fp := range fingerprints { + if strings.TrimSpace(fp.Name) == "" { + return fmt.Errorf("ns fingerprint[%d] missing name", i) + } + + if len(fp.HostPatterns) == 0 { + return fmt.Errorf("ns fingerprint[%d] has no host patterns", i) + } + + for j, pattern := range fp.HostPatterns { + if strings.TrimSpace(pattern) == "" { + return fmt.Errorf("ns fingerprint[%d] has empty host pattern at index %d", i, j) + } + } + } + + return nil +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints.go b/scanners/subdomain-takeover/internal/detect/fingerprints.go index b7098341a2..d1491995cd 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprints.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprints.go @@ -3,12 +3,23 @@ package detect import "strings" type CNAMEProviderFingerprint struct { - Cname []string - Name string - Nxdomain bool - Fingerprint string + Cname []string `json:"cname"` + Name string `json:"name"` + Nxdomain bool `json:"nxdomain"` + Fingerprint string `json:"fingerprint"` + Mode FingerprintMode `json:"mode,omitempty"` } +type FingerprintMode string + +const ( + FingerprintModeLiteral FingerprintMode = "literal" + FingerprintModeRegex FingerprintMode = "regex" +) + +var CNAMEProviderFingerprints []CNAMEProviderFingerprint +var NSProviderFingerprints []NSProviderFingerprint + type NSProviderStatus string const ( @@ -20,12 +31,12 @@ const ( ) type NSProviderFingerprint struct { - Name string - ProviderURL string - Status NSProviderStatus - HostPatterns []string - InstructionsURL string - PrivateDNS bool + Name string `json:"name"` + ProviderURL string `json:"provider_url"` + Status NSProviderStatus `json:"status"` + HostPatterns []string `json:"host_patterns"` + InstructionsURL string `json:"instructions_url,omitempty"` + PrivateDNS bool `json:"private_dns,omitempty"` } func (f *NSProviderFingerprint) ContainsNSHost(host string) bool { @@ -75,285 +86,6 @@ func wildcardHostMatch(pattern, host string) bool { return true } -var NSProviderFingerprints = []NSProviderFingerprint{ - { - Name: "000Domains", - ProviderURL: "https://000domains.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.000domains.com", "ns2.000domains.com", "fwns1.000domains.com", "fwns2.000domains.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/19", - }, - { - Name: "AWS Route 53", - ProviderURL: "https://aws.amazon.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns-*.awsdns-*.org", "ns-*.awsdns-*.co.uk", "ns-*.awsdns-*.com", "ns-*.awsdns-*.net"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/1", - }, - { - Name: "Azure (Microsoft)", - ProviderURL: "https://azure.microsoft.com/", - Status: NSStatusEdgeCase, - HostPatterns: []string{"ns1-*.azure-dns.com", "ns2-*.azure-dns.net", "ns3-*.azure-dns.org", "ns4-*.azure-dns.info"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/5", - }, - { - Name: "BigCommerce", - ProviderURL: "https://bigcommerce.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.bigcommerce.com", "ns2.bigcommerce.com", "ns3.bigcommerce.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/35", - }, - { - Name: "Bizland", - ProviderURL: "https://bizland.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.bizland.com", "ns2.bizland.com", "clickme.click2site.com", "clickme2.click2site.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/3", - }, - { - Name: "ClouDNS", - ProviderURL: "https://cloudns.net/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"*.cloudns.net"}, - }, - { - Name: "Cloudflare", - ProviderURL: "https://cloudflare.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"*.ns.cloudflare.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/10", - }, - { - Name: "Digital Ocean", - ProviderURL: "https://digitalocean.com/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.digitalocean.com", "ns2.digitalocean.com", "ns3.digitalocean.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/22", - }, - { - Name: "DNSMadeEasy", - ProviderURL: "https://dnsmadeeasy.com/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns*.dnsmadeeasy.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/6", - }, - { - Name: "DNSimple", - ProviderURL: "https://dnsimple.com/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.dnsimple.com", "ns2.dnsimple.com", "ns3.dnsimple.com", "ns4.dnsimple.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/16", - }, - { - Name: "Domain.com", - ProviderURL: "https://domain.com/", - Status: NSStatusVulnerableWithPurchase, - HostPatterns: []string{"ns1.domain.com", "ns2.domain.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/17", - }, - { - Name: "DomainPeople", - ProviderURL: "https://domainpeople.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.domainpeople.com", "ns2.domainpeople.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/14", - }, - { - Name: "Dotster", - ProviderURL: "https://dotster.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.dotster.com", "ns2.dotster.com", "ns1.nameresolve.com", "ns2.nameresolve.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/18", - }, - { - Name: "Dreamhost", - ProviderURL: "https://dreamhost.com/", - Status: NSStatusEdgeCase, - HostPatterns: []string{"ns1.dreamhost.com", "ns2.dreamhost.com", "ns3.dreamhost.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/40", - }, - { - Name: "EasyDNS", - ProviderURL: "https://easydns.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"dns1.easydns.com", "dns2.easydns.net", "dns3.easydns.org", "dns4.easydns.info"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/9", - }, - { - Name: "Gandi.net", - ProviderURL: "https://gandi.net/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"a.dns.gandi.net", "b.dns.gandi.net", "c.dns.gandi.net"}, - }, - { - Name: "Google Cloud", - ProviderURL: "https://cloud.google.com/", - Status: NSStatusEdgeCase, - HostPatterns: []string{"ns-cloud-*.googledomains.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/2", - }, - { - Name: "Hostinger (old NS)", - ProviderURL: "https://hostinger.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.hostinger.com", "ns2.hostinger.com"}, - }, - { - Name: "Hover", - ProviderURL: "https://hover.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.hover.com", "ns2.hover.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/21", - }, - { - Name: "Hurricane Electric", - ProviderURL: "https://dns.he.net/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.he.net", "ns2.he.net", "ns3.he.net", "ns4.he.net", "ns5.he.net"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/25", - }, - { - Name: "Linode", - ProviderURL: "https://linode.com/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.linode.com", "ns2.linode.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/26", - }, - { - Name: "MediaTemple (mt)", - ProviderURL: "https://mediatemple.net/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.mediatemple.net", "ns2.mediatemple.net"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/23", - }, - { - Name: "MyDomain", - ProviderURL: "https://mydomain.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns1.mydomain.com", "ns2.mydomain.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/4", - }, - { - Name: "Name.com", - ProviderURL: "https://name.com/", - Status: NSStatusVulnerableWithPurchase, - HostPatterns: []string{"ns1*.name.com", "ns2*.name.com", "ns3*.name.com", "ns4*.name.com"}, - InstructionsURL: "https://github.com/libertalialtd/can-i-take-over-dns/issues/8", - }, - { - Name: "Namecheap", - ProviderURL: "https://namecheap.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"*.namecheaphosting.com", "*.registrar-servers.com"}, - }, - { - Name: "Network Solutions", - ProviderURL: "https://networksolutions.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.worldnic.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/15", - }, - { - Name: "NS1", - ProviderURL: "https://nsone.net/", - Status: NSStatusRegistrationClosed, - HostPatterns: []string{"dns1.p*.nsone.net", "dns2.p*.nsone.net", "dns3.p*.nsone.net", "dns4.p*.nsone.net"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/7", - }, - { - Name: "TierraNet", - ProviderURL: "https://tierra.net/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.domaindiscover.com", "ns2.domaindiscover.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/24", - }, - { - Name: "Reg.ru", - ProviderURL: "https://reg.ru/", - Status: NSStatusVulnerable, - HostPatterns: []string{"ns1.reg.ru", "ns2.reg.ru"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/28", - }, - { - Name: "UltraDNS", - ProviderURL: "https://www.home.neustar/dns-services/ultra-dns", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"pdns*.ultradns.com", "udns*.ultradns.com", "sdns*.ultradns.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/29", - }, - { - Name: "Yahoo Small Business", - ProviderURL: "https://yahoosmallbusiness.com/", - Status: NSStatusVulnerableWithPurchase, - HostPatterns: []string{"yns1.yahoo.com", "yns2.yahoo.com"}, - InstructionsURL: "https://github.com/indianajson/can-i-take-over-dns/issues/20", - }, - { - Name: "Activision", - ProviderURL: "https://activision.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.activision.com"}, - PrivateDNS: true, - }, - { - Name: "Adobe", - ProviderURL: "https://adobe.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"adobe-dns-0*.adobe.com"}, - PrivateDNS: true, - }, - { - Name: "Apple", - ProviderURL: "https://apple.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"a.ns.apple.com", "b.ns.apple.com", "c.ns.apple.com", "d.ns.apple.com"}, - PrivateDNS: true, - }, - { - Name: "Automattic", - ProviderURL: "https://automattic.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.automattic.com"}, - PrivateDNS: true, - }, - { - Name: "Capital One", - ProviderURL: "https://capitalone.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.capitalone.com"}, - PrivateDNS: true, - }, - { - Name: "Disney", - ProviderURL: "https://disney.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.twdcns.com", "ns*.twdcns.info", "ns*.twdcns.co.uk"}, - PrivateDNS: true, - }, - { - Name: "Google", - ProviderURL: "https://google.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns*.google.com"}, - PrivateDNS: true, - }, - { - Name: "Lowe's", - ProviderURL: "https://lowes.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"authns*.lowes.com"}, - PrivateDNS: true, - }, - { - Name: "T-Mobile", - ProviderURL: "https://tmobileus.com/", - Status: NSStatusNotVulnerable, - HostPatterns: []string{"ns10.tmobileus.com", "ns10.tmobileus.net"}, - PrivateDNS: true, - }, -} - func (f *CNAMEProviderFingerprint) ContainsTarget(target string) bool { for _, cname := range f.Cname { if strings.HasSuffix(target, cname) { @@ -363,230 +95,14 @@ func (f *CNAMEProviderFingerprint) ContainsTarget(target string) bool { return false } -var CNAMEProviderFingerprints = []CNAMEProviderFingerprint{ - { - Cname: []string{ - "elasticbeanstalk.com", - }, - Nxdomain: true, - Name: "AWS/Elastic Beanstalk", - Fingerprint: "NXDOMAIN", - }, - { - Cname: []string{ - "s3.amazonaws.com", - }, - Nxdomain: false, - Name: "AWS/S3", - Fingerprint: "The specified bucket does not exist", - }, - { - Cname: []string{ - "agilecrm.com", - }, - Nxdomain: false, - Name: "Agile CRM", - Fingerprint: "Sorry, this page is no longer available.", - }, - { - Cname: []string{ - "airee.ru", - }, - Nxdomain: false, - Name: "Airee.ru", - Fingerprint: "Ошибка 402. Сервис Айри.рф не оплачен", - }, - { - Cname: []string{ - "animaapp.io", - }, - Nxdomain: false, - Name: "Anima", - Fingerprint: "The page you were looking for does not exist.", - }, - { - Cname: []string{ - "bitbucket.io", - }, - Nxdomain: false, - Name: "Bitbucket", - Fingerprint: "Repository not found", - }, - { - Cname: []string{ - "trydiscourse.com", - }, - Nxdomain: true, - Name: "Discourse", - Fingerprint: "NXDOMAIN", - }, - { - Cname: []string{ - "furyns.com", - }, - Nxdomain: false, - Name: "Gemfury", - Fingerprint: "404: This page could not be found.", - }, - { - Cname: []string{ - "ghost.io", - }, - Nxdomain: false, - Name: "Ghost", - Fingerprint: "Site unavailable\\.|Failed to resolve DNS path for this host", - }, - { - Cname: []string{ - "hatenablog.com", - }, - Nxdomain: false, - Name: "HatenaBlog", - Fingerprint: "404 Blog is not found", - }, - { - Cname: []string{ - "helpjuice.com", - }, - Nxdomain: false, - Name: "Help Juice", - Fingerprint: "We could not find what you're looking for.", - }, - { - Cname: []string{ - "helpscoutdocs.com", - }, - Nxdomain: false, - Name: "Help Scout", - Fingerprint: "No settings were found for this company:", - }, - { - Cname: []string{ - "helprace.com", - }, - Nxdomain: false, - Name: "Helprace", - Fingerprint: "HTTP_STATUS=301", - }, - { - Cname: []string{ - "youtrack.cloud", - }, - Nxdomain: false, - Name: "JetBrains", - Fingerprint: "is not a registered InCloud YouTrack", - }, - { - Cname: []string{ - "launchrock.com", - }, - Nxdomain: false, - Name: "LaunchRock", - Fingerprint: "HTTP_STATUS=500", - }, - { - Cname: []string{ - "cloudapp.azure.com", - "azurewebsites.net", - "blob.core.windows.net", - "cloudapp.azure.com", - "azure-api.net", - "azurehdinsight.net", - "azureedge.net", - "azurecontainer.io", - "database.windows.net", - "azuredatalakestore.net", - "search.windows.net", - "azurecr.io", - "redis.cache.windows.net", - "azurehdinsight.net", - "servicebus.windows.net", - "trafficmanager.net", - "visualstudio.com", - }, - Nxdomain: true, - Name: "Microsoft Azure", - Fingerprint: "NXDOMAIN", - }, - { - Cname: []string{ - "ngrok.io", - }, - Nxdomain: false, - Name: "Ngrok", - Fingerprint: "Tunnel .*.ngrok.io not found", - }, - { - Cname: []string{ - "readme.io", - }, - Nxdomain: false, - Name: "Readme.io", - Fingerprint: "The creators of this project are still working on making everything perfect!", - }, - { - Cname: []string{ - "52.16.160.97", - }, - Nxdomain: false, - Name: "SmartJobBoard", - Fingerprint: "This job board website is either expired or its domain name is invalid.", - }, - { - Cname: []string{ - "s.strikinglydns.com", - }, - Nxdomain: false, - Name: "Strikingly", - Fingerprint: "PAGE NOT FOUND.", - }, - { - Cname: []string{ - "na-west1.surge.sh", - }, - Nxdomain: false, - Name: "Surge.sh", - Fingerprint: "project not found", - }, - { - Cname: []string{ - "surveysparrow.com", - }, - Nxdomain: false, - Name: "SurveySparrow", - Fingerprint: "Account not found.", - }, - { - Cname: []string{ - "read.uberflip.com", - }, - Nxdomain: false, - Name: "Uberflip", - Fingerprint: "The URL you've accessed does not provide a hub.", - }, - { - Cname: []string{ - "stats.uptimerobot.com", - }, - Nxdomain: false, - Name: "Uptimerobot", - Fingerprint: "page not found", - }, - { - Cname: []string{ - "wordpress.com", - }, - Nxdomain: false, - Name: "Wordpress", - Fingerprint: "Do you want to register .*.wordpress.com?", - }, - { - Cname: []string{ - "worksites.net", - "69.164.223.206", - }, - Nxdomain: false, - Name: "Worksites", - Fingerprint: "Hello! Sorry, but the website you’re looking for doesn’t exist.", - }, +func normalizeFingerprintMode(mode FingerprintMode, fingerprint string) FingerprintMode { + if mode == FingerprintModeLiteral || mode == FingerprintModeRegex { + return mode + } + + if strings.Contains(fingerprint, ".*") || strings.Contains(fingerprint, "\\") { + return FingerprintModeRegex + } + + return FingerprintModeLiteral } From 64e789c7376985bcdebebfcd4b0e21d5a8c5f0b6 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:18:51 -0300 Subject: [PATCH 22/65] remove remediation.go file to serve translated text in api/frontend --- scanners/subdomain-takeover/internal/detect/remediation.go | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 scanners/subdomain-takeover/internal/detect/remediation.go diff --git a/scanners/subdomain-takeover/internal/detect/remediation.go b/scanners/subdomain-takeover/internal/detect/remediation.go deleted file mode 100644 index 9d577baa7f..0000000000 --- a/scanners/subdomain-takeover/internal/detect/remediation.go +++ /dev/null @@ -1,5 +0,0 @@ -package detect - -func RemediationForReason(reasonCode string, provider string, lameType string) { - return -} From 2c873aba90bd4c635c5ef6fef467c97388bd27f6 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:19:47 -0300 Subject: [PATCH 23/65] add RecordType enums --- scanners/subdomain-takeover/internal/model/finding.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go index 5d387c0f1d..57dbc83c67 100644 --- a/scanners/subdomain-takeover/internal/model/finding.go +++ b/scanners/subdomain-takeover/internal/model/finding.go @@ -1,9 +1,16 @@ package model +type RecordType string + +const ( + RecordTypeCNAME RecordType = "CNAME" + RecordTypeNS RecordType = "NS" +) + type Finding struct { Domain string `json:"domain"` DomainKey string `json:"domain_key"` - RecordType string `json:"record_type"` + RecordType RecordType `json:"record_type"` Target string `json:"target"` Provider string `json:"provider"` LameType string `json:"lame_type"` From f405a61604273509af31c3c86e0c547df5cd0695 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:20:11 -0300 Subject: [PATCH 24/65] add confidence and reason code enums --- .../internal/detect/confidence.go | 32 +++++++++++++++++++ .../internal/detect/reason_codes.go | 15 +++++++++ 2 files changed, 47 insertions(+) create mode 100644 scanners/subdomain-takeover/internal/detect/reason_codes.go diff --git a/scanners/subdomain-takeover/internal/detect/confidence.go b/scanners/subdomain-takeover/internal/detect/confidence.go index ad6c03d18d..9865a561db 100644 --- a/scanners/subdomain-takeover/internal/detect/confidence.go +++ b/scanners/subdomain-takeover/internal/detect/confidence.go @@ -1 +1,33 @@ package detect + +const ( + ConfidenceSuspected = "suspected" + ConfidenceProbable = "probable" + ConfidenceConfirmed = "confirmed" +) + +// ConfidenceForReason maps stable reason codes to confidence levels. +// Unknown reason codes default to suspected. +func ConfidenceForReason(reasonCode ReasonCode) string { + switch reasonCode { + case ReasonCNAMEDanglingNXDOMAIN: + return ConfidenceProbable + case ReasonCNAMEProviderFingerprintBodyMatch: + return ConfidenceProbable + case ReasonCNAMETargetMatchMissingNXDOMAIN: + return ConfidenceSuspected + case ReasonCNAMETargetMatchMissingBodyFP: + return ConfidenceSuspected + + case ReasonNSFullLameProviderVulnerable: + return ConfidenceConfirmed + case ReasonNSPartialLameProviderVulnerable: + return ConfidenceProbable + case ReasonNSLameProviderUnknown: + return ConfidenceSuspected + case ReasonNSProviderMatchOnly: + return ConfidenceSuspected + default: + return ConfidenceSuspected + } +} diff --git a/scanners/subdomain-takeover/internal/detect/reason_codes.go b/scanners/subdomain-takeover/internal/detect/reason_codes.go new file mode 100644 index 0000000000..f080b18329 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/reason_codes.go @@ -0,0 +1,15 @@ +package detect + +type ReasonCode string + +const ( + ReasonCNAMEDanglingNXDOMAIN ReasonCode = "CNAME_DANGLING_NXDOMAIN" + ReasonCNAMEProviderFingerprintBodyMatch ReasonCode = "CNAME_PROVIDER_FINGERPRINT_BODY_MATCH" + ReasonCNAMETargetMatchMissingNXDOMAIN ReasonCode = "CNAME_TARGET_MATCH_MISSING_NXDOMAIN" + ReasonCNAMETargetMatchMissingBodyFP ReasonCode = "CNAME_TARGET_MATCH_MISSING_BODY_FINGERPRINT" + + ReasonNSFullLameProviderVulnerable ReasonCode = "NS_FULL_LAME_PROVIDER_VULNERABLE" + ReasonNSPartialLameProviderVulnerable ReasonCode = "NS_PARTIAL_LAME_PROVIDER_VULNERABLE" + ReasonNSLameProviderUnknown ReasonCode = "NS_LAME_PROVIDER_UNKNOWN" + ReasonNSProviderMatchOnly ReasonCode = "NS_PROVIDER_MATCH_ONLY" +) From d18030718304344f9b109a458eea3bab09c5f0a2 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:20:45 -0300 Subject: [PATCH 25/65] move http interface for NS lookups to own file --- .../internal/detect/body_matcher.go | 65 +++++++++++++++++++ .../internal/detect/rules.go | 48 +++----------- 2 files changed, 75 insertions(+), 38 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/body_matcher.go diff --git a/scanners/subdomain-takeover/internal/detect/body_matcher.go b/scanners/subdomain-takeover/internal/detect/body_matcher.go new file mode 100644 index 0000000000..622b2f7c51 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/body_matcher.go @@ -0,0 +1,65 @@ +package detect + +import ( + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" +) + +type BodyFingerprintMatcher interface { + Contains(domain string, fingerprint string, mode FingerprintMode) bool +} + +type HTTPBodyFingerprintMatcher struct { + client *http.Client +} + +func NewHTTPBodyFingerprintMatcher(timeout time.Duration) *HTTPBodyFingerprintMatcher { + return &HTTPBodyFingerprintMatcher{ + client: &http.Client{Timeout: timeout}, + } +} + +type NoopBodyFingerprintMatcher struct{} + +func NewNoopBodyFingerprintMatcher() *NoopBodyFingerprintMatcher { + return &NoopBodyFingerprintMatcher{} +} + +func (m *NoopBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { + return false +} + +func (m *HTTPBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { + if m == nil || m.client == nil { + return false + } + + url := fmt.Sprintf("http://%s", domain) + res, err := m.client.Get(url) + if err != nil { + return false + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return false + } + + bodyText := string(body) + resolvedMode := normalizeFingerprintMode(mode, fingerprint) + + if resolvedMode == FingerprintModeRegex { + re, err := regexp.Compile(fingerprint) + if err != nil { + return false + } + return re.MatchString(bodyText) + } + + return strings.Contains(bodyText, fingerprint) +} diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index 9cb5ac5738..44626a372b 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -1,28 +1,21 @@ package detect -import ( - "fmt" - "io" - "net/http" - "strings" -) - type CNAMEHit struct { Matched bool Provider string - ReasonCode string + ReasonCode ReasonCode NeedsNX bool } type NSHit struct { Matched bool Provider string - ReasonCode string + ReasonCode ReasonCode ProviderVulnerable bool RegistrarMismatch bool } -func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint) *CNAMEHit { +func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { for _, fp := range fingerprints { if fp.ContainsTarget(evidence.Target) { hit := CNAMEHit{ @@ -32,16 +25,17 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider NeedsNX: fp.Nxdomain, } if hit.NeedsNX { - hit.ReasonCode = "CNAME_TARGET_MATCH_MISSING_NXDOMAIN" - if evidence.QueryAnswer.A == "NXDOMAIN" { + hit.ReasonCode = ReasonCNAMETargetMatchMissingNXDOMAIN + if evidence.NoResolve { hit.Matched = true - hit.ReasonCode = "CNAME_DANGLING_NXDOMAIN" + hit.ReasonCode = ReasonCNAMEDanglingNXDOMAIN } } else { - hit.ReasonCode = "CNAME_TARGET_MATCH_MISSING_BODY_FINGERPRINT" - if urlBodyContainsFingerprint(evidence.Domain, fp.Fingerprint) { + hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP + mode := normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + if matcher != nil && matcher.Contains(evidence.Domain, fp.Fingerprint, mode) { hit.Matched = true - hit.ReasonCode = "CNAME_PROVIDER_FINGERPRINT_BODY_MATCH" + hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch } } return &hit @@ -67,25 +61,3 @@ func ShouldEmitCNAME(hit *CNAMEHit) bool { func ShouldEmitNSHijack(evidence NSEvidence, hit *NSHit) bool { return false } - -func urlBodyContainsFingerprint(domain string, fp string) bool { - // curl domain and look for matching fp in body - url := fmt.Sprintf("http://%s", domain) - res, err := http.Get(url) - if err != nil { - // log error - return false - } - defer res.Body.Close() - - body, err := io.ReadAll(res.Body) - if err != nil { - // log err - return false - } - - if strings.Contains(string(body), fp) { - return true - } - return false -} From ad7c2729c124905f932f7eb211e733b7edfb0a59 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:21:55 -0300 Subject: [PATCH 26/65] switch from using QueryAnswers struct field to NoResolve bool for CNAME evidence --- .../internal/detect/evidence.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index fb40318cfc..ba54926bb8 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -8,9 +8,9 @@ import ( // inputs from evidence.go type CNAMEEvidence struct { - Domain string - Target string - QueryAnswer model.QueryAnswers // or just needed fields + Domain string + Target string + NoResolve bool } type NSEvidence struct { @@ -26,9 +26,9 @@ func ExtractCNAMEEvidence(input model.Input) *CNAMEEvidence { } return &CNAMEEvidence{ - Domain: input.Domain, - Target: parseCname(*input.CnameRecord), - QueryAnswer: input.QueryAnswers, + Domain: input.Domain, + Target: parseCname(*input.CnameRecord), + NoResolve: input.ResolveChain == nil, } } @@ -46,6 +46,6 @@ func ClassifyLameType(nsChecks []model.NsCheck) { func parseCname(record string) string { trimmed := strings.Trim(record, ".") lower := strings.ToLower(trimmed) - record_tokens := strings.Split(lower, " ") - return record_tokens[len(record_tokens)-1] + recordTokens := strings.Split(lower, " ") + return recordTokens[len(recordTokens)-1] } From 967a0882a87a96e7a8499551667342aa36e65010 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:23:05 -0300 Subject: [PATCH 27/65] add contructor for Classifier --- .../internal/detect/classifier.go | 61 ++++++++++++------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index 0caea61196..28fb59649b 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -2,33 +2,48 @@ package detect import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" -func Classify(input model.Input) ([]model.Finding, error) { +type Classifier struct { + Matcher BodyFingerprintMatcher +} + +func NewClassifier(matcher BodyFingerprintMatcher) *Classifier { + return &Classifier{Matcher: matcher} +} + +func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { + matcher := c.Matcher + if matcher == nil { + matcher = NewNoopBodyFingerprintMatcher() + } + + return Classify(input, matcher) +} + +func Classify(input model.Input, matcher BodyFingerprintMatcher) ([]model.Finding, error) { findings := []model.Finding{} cnameEvidence := ExtractCNAMEEvidence(input) - cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints) - if ShouldEmitCNAME(cnameHit) { - // compute confidence - // compute reason/remediation - findings = append(findings, model.Finding{ - Domain: cnameEvidence.Domain, - RecordType: "CNAME", - Target: cnameEvidence.Target, - Provider: cnameHit.Provider, - ReasonCode: cnameHit.ReasonCode, - Confidence: "", - Remediation: "", - }) + if cnameEvidence != nil { + cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints, matcher) + if ShouldEmitCNAME(cnameHit) { + findings = append(findings, model.Finding{ + Domain: cnameEvidence.Domain, + RecordType: model.RecordTypeCNAME, + Target: cnameEvidence.Target, + Provider: cnameHit.Provider, + ReasonCode: string(cnameHit.ReasonCode), + Confidence: ConfidenceForReason(cnameHit.ReasonCode), + }) + } } - nsEvidence := ExtractNSEvidence(input) - nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) - if ShouldEmitNSHijack(*nsEvidence, nsHit) { - // compute confidence - // compute reason/remediation - findings = append(findings, model.Finding{ - Domain: input.Domain, - }) - } + // nsEvidence := ExtractNSEvidence(input) + // nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) + // if ShouldEmitNSHijack(*nsEvidence, nsHit) { + // findings = append(findings, model.Finding{ + // Domain: input.Domain, + // }) + // } + return findings, nil } From 92a05793c311af658b10577d8c418277e8d1ff90 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 16:23:15 -0300 Subject: [PATCH 28/65] update README.md --- scanners/subdomain-takeover/README.md | 117 +++++++++----------------- 1 file changed, 39 insertions(+), 78 deletions(-) diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md index 0bfcdaf9ba..640b246776 100644 --- a/scanners/subdomain-takeover/README.md +++ b/scanners/subdomain-takeover/README.md @@ -5,10 +5,17 @@ Go microservice that consumes DNS scan results and emits normalized subdomain ta ## What it does 1. Consumes from `scans.dns_scanner_results` (JetStream stream: `SCANS`) -2. Evaluates takeover risk using passive DNS evidence + provider fingerprints +2. Evaluates passive CNAME takeover risk using provider fingerprints 3. Assigns confidence (`suspected`, `probable`, `confirmed`) 4. Publishes findings to `scans.findings.upsert` +## Current detection scope + +- Implemented: CNAME-based passive detection +- Not implemented yet: NS-based (Sitting Duck) detection + +The service currently focuses on passive CNAME classification only. NS-related types and reason codes exist in the codebase for upcoming work, but NS findings are not emitted yet. + ## Passive-only detection policy This service uses passive checks only. @@ -19,63 +26,36 @@ As a result, confidence is conservative and evidence-driven: - `probable`: strong passive indicators with known exploitable conditions - `confirmed`: reserved for deterministic passive signatures only -## Sitting Duck DNS hijack coverage +## Confidence model (current) -The scanner includes passive detection for Sitting Duck style DNS hijack risk. +Current production behavior for CNAME findings: -A Sitting Duck takeover is considered possible when all of the following are true: - -1. The registered domain uses or delegates authoritative DNS services to a provider other than the registrar. -2. The authoritative name server cannot resolve authoritatively for the domain (lame delegation). -3. The DNS provider is known to allow domain/zone claiming without strong ownership verification. +- `suspected` + - provider target matched, but required passive signal is missing +- `probable` + - dangling target with NXDOMAIN where required, or + - provider fingerprint body match for providers that require body verification +- `confirmed` + - reserved; not currently emitted by CNAME rules -### Variations covered +## Fingerprint data -- Partially lame delegation: some NS records are lame, others still answer. -- Full lame delegation: all authoritative NS are lame. -- Redelegation drift: domain appears redelegated while stale or invalid NS remains and creates takeover exposure. +Provider fingerprint data is stored in: -If lame delegation and exploitable provider conditions are both present, the domain is considered hijackable risk. +- `internal/detect/data/cname_fingerprints.json` +- `internal/detect/data/ns_fingerprints.json` -## Confidence model +The service embeds and validates these files at startup (`detect.LoadFingerprints`). -This model applies across both CNAME-based takeover checks and NS-based Sitting Duck checks. +For CNAME body matching: -- `suspected` - - dangling/suspicious DNS pattern only, or - - lame delegation evidence without known exploitable-provider confirmation -- `probable` - - DNS pattern + known provider unclaimed signature, or - - lame delegation confirmed (`partial` or `full`) + exploitable-provider rule match -- `confirmed` - - explicit passive takeover indicator with deterministic evidence - - if deterministic passive proof is missing, cap at `probable` - -## Required evidence for findings - -Each finding should include: -- `domain` -- `domain_key` -- `record_type` (`CNAME` or `NS`) -- `target` or `ns_host` -- `provider` -- `lame_type` (`partial` or `full`) for Sitting Duck findings -- per-NS resolver outcomes where applicable -- `confidence` -- `reason_code` -- remediation guidance - -## Remediation guidance - -Typical remediation: -1. Fix delegation at registrar and authoritative DNS provider. -2. Remove stale or dead NS entries. -3. Reclaim or securely rebind abandoned DNS zones or accounts. -4. Validate ownership controls for DNS provider account workflows. +- `mode: literal` performs plain substring search +- `mode: regex` performs regular expression matching +- if `mode` is omitted, the loader applies backward-compatible inference ## Prerequisites -- Go 1.24+ +- Go 1.25+ - NATS with JetStream enabled - `nats` CLI (optional, for local testing) @@ -95,43 +75,24 @@ Environment variables: - `WORKER_COUNT` (default: `5`) - `LOG_LEVEL` (default: `info`) -Local test flow: - -Publish a test request: -```bash -nats pub scans.dns_scanner_results '{"domain":"example.gc.ca","domain_key":"12345","shared_id":null,"results":{"record_exists":true,"rcode":"NOERROR","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_records":{"hostnames":["ns1.example-dns-provider.net"],"warnings":[]}}}' -``` +## Message ack behavior -Watch findings: -```bash -nats sub "scans.findings.upsert" -``` +The worker uses explicit JetStream ack semantics: -## Development milestones +- decode failures -> `Term()` (drop poison message) +- classify/publish failures -> `Nak()` (retryable) +- successful processing -> `Ack()` -1. Build config + NATS consume/publish loop. -2. Add DNS evidence extraction. -3. Add provider fingerprint rule engine. -4. Add confidence scoring and remediation text. -5. Add tests for classification rules and message handling. +## Local test flow ---- +Publish a test request: -## External Go modules to install +```bash +nats pub scans.dns_scanner_results '{"domain":"example.gc.ca","domain_key":"12345","base_domain":"example.gc.ca","zone_apex":"example.gc.ca","record_exists":true,"rcode":"NOERROR","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_records":{"hostnames":["ns1.example-dns-provider.net"],"warnings":[]}}' +``` -Start minimal: +Watch findings: ```bash -go get github.com/nats-io/nats.go -go get github.com/miekg/dns -go get github.com/rs/zerolog -go get golang.org/x/sync/errgroup -go get github.com/stretchr/testify +nats sub "scans.findings.upsert" ``` - -What each is for: -- `nats.go`: JetStream consume/publish -- `miekg/dns`: robust DNS lookups and record parsing -- `zerolog`: structured logs without heavy setup -- `errgroup`: clean goroutine lifecycle management -- `testify`: easier assertions in unit tests From 9ba9e1fd9bebf2e4aa7390f46a51849e7ad686e2 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 17:04:05 -0300 Subject: [PATCH 29/65] remove unneeded scan result fixtures --- .../fixtures/cyber-gc-ca_dns_scan.json | 202 ------------------ .../fixtures/www-cyber-gc-ca_dns_scan.json | 23 -- 2 files changed, 225 deletions(-) delete mode 100644 scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json delete mode 100644 scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json diff --git a/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json b/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json deleted file mode 100644 index cad720cc93..0000000000 --- a/scanners/subdomain-takeover/fixtures/cyber-gc-ca_dns_scan.json +++ /dev/null @@ -1,202 +0,0 @@ -{ - "domain": "cyber.gc.ca", - "base_domain": "cyber.gc.ca", - "zone_apex": "cyber.gc.ca", - "record_exists": true, - "rcode": "NOERROR", - "resolve_chain": [ - [ - "cyber.gc.ca. 7161 IN A 20.151.96.73" - ] - ], - "resolve_ips": [ - "20.151.96.73" - ], - "cname_record": null, - "mx_records": { - "hosts": [ - { - "preference": 10, - "hostname": "cyber-gc-ca.mail.protection.outlook.com", - "addresses": [ - "52.101.190.1", - "52.101.192.0", - "52.101.192.1", - "52.101.190.0" - ] - } - ], - "warnings": [] - }, - "ns_records": { - "hostnames": [ - "ns1.d-zone.ca", - "ns2.d-zone.ca", - "ns40.ent.global.gc.ca", - "ns41.ent.global.gc.ca", - "ns11.ent.global.gc.ca", - "ns10.ent.global.gc.ca" - ], - "warnings": [] - }, - "dkim": {}, - "spf": { - "record": "v=spf1 ip4:40.80.253.57 a:smtp.ops.cyber.gc.ca a:edge.cyber.gc.ca include:spf.protection.outlook.com -all", - "valid": true, - "dns_void_lookups": 0, - "warnings": [], - "parsed": { - "pass": [ - { - "value": "40.80.253.57", - "mechanism": "ip4" - }, - { - "value": "52.139.3.76", - "mechanism": "a" - }, - { - "value": "205.193.218.114", - "mechanism": "a" - }, - { - "value": "205.193.218.115", - "mechanism": "a" - } - ], - "neutral": [], - "softfail": [], - "fail": [], - "include": [ - { - "domain": "spf.protection.outlook.com", - "record": "v=spf1 ip4:40.92.0.0/15 ip4:40.107.0.0/16 ip4:52.100.0.0/15 ip4:52.102.0.0/16 ip4:52.103.0.0/17 ip4:104.47.0.0/17 ip6:2a01:111:f400::/48 ip6:2a01:111:f403::/49 ip6:2a01:111:f403:8000::/51 ip6:2a01:111:f403:c000::/51 ip6:2a01:111:f403:f000::/52 -all", - "dns_void_lookups": 0, - "parsed": { - "pass": [ - { - "value": "40.92.0.0/15", - "mechanism": "ip4" - }, - { - "value": "40.107.0.0/16", - "mechanism": "ip4" - }, - { - "value": "52.100.0.0/15", - "mechanism": "ip4" - }, - { - "value": "52.102.0.0/16", - "mechanism": "ip4" - }, - { - "value": "52.103.0.0/17", - "mechanism": "ip4" - }, - { - "value": "104.47.0.0/17", - "mechanism": "ip4" - }, - { - "value": "2a01:111:f400::/48", - "mechanism": "ip6" - }, - { - "value": "2a01:111:f403::/49", - "mechanism": "ip6" - }, - { - "value": "2a01:111:f403:8000::/51", - "mechanism": "ip6" - }, - { - "value": "2a01:111:f403:c000::/51", - "mechanism": "ip6" - }, - { - "value": "2a01:111:f403:f000::/52", - "mechanism": "ip6" - } - ], - "neutral": [], - "softfail": [], - "fail": [], - "include": [], - "redirect": null, - "exp": null, - "duplicate_include": [] - }, - "warnings": [], - "spf_default": "fail", - "lookups": 0 - } - ], - "redirect": null, - "exp": null, - "duplicate_include": [] - }, - "spf_default": "fail", - "lookups": 3 - }, - "dmarc": { - "record": "v=DMARC1; p=reject; pct=100; rua=mailto:dmarc@cyber.gc.ca", - "valid": true, - "location": "cyber.gc.ca", - "warnings": [], - "tags": { - "v": { - "value": "DMARC1", - "explicit": true - }, - "p": { - "value": "reject", - "explicit": true - }, - "pct": { - "value": 100, - "explicit": true - }, - "rua": { - "value": [ - { - "scheme": "mailto", - "address": "dmarc@cyber.gc.ca", - "size_limit": null, - "accepting": true - } - ], - "explicit": true - }, - "adkim": { - "value": "r", - "explicit": false - }, - "aspf": { - "value": "r", - "explicit": false - }, - "fo": { - "value": "0", - "explicit": false - }, - "rf": { - "value": "afrf", - "explicit": false - }, - "ri": { - "value": 86400, - "explicit": false - }, - "sp": { - "value": "reject", - "explicit": false - } - }, - "effective_policy_source": "p", - "effective_policy": "reject" - }, - "wildcard_sibling": false, - "wildcard_entry": false, - "zone_dnssec_enabled": null -} diff --git a/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json b/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json deleted file mode 100644 index 8f546a74a5..0000000000 --- a/scanners/subdomain-takeover/fixtures/www-cyber-gc-ca_dns_scan.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "domain": "www.cyber.gc.ca", - "base_domain": "cyber.gc.ca", - "zone_apex": "www.cyber.gc.ca", - "record_exists": true, - "rcode": "NOERROR", - "resolve_chain": [ - ["www.cyber.gc.ca. 5550 IN CNAME cyber.gc.ca."], - ["cyber.gc.ca. 5550 IN A 20.151.96.73"] - ], - "cname_record": "www.cyber.gc.ca. 5550 IN CNAME cyber.gc.ca.", - "ns_records": { - "hostnames": [ - "ns40.ent.global.gc.ca", - "ns10.ent.global.gc.ca", - "ns41.ent.global.gc.ca", - "ns11.ent.global.gc.ca", - "ns2.d-zone.ca", - "ns1.d-zone.ca" - ], - "warnings": [] - } -} From 16dc260e13ba15f1899b3403ea1799ad898e3cf0 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 10 Jul 2026 17:05:11 -0300 Subject: [PATCH 30/65] update model.input to take in published dns_scanner_results payload --- .../internal/detect/classifier.go | 3 +- .../internal/detect/evidence.go | 17 ++++---- .../internal/messaging/consumer.go | 1 - .../internal/model/finding.go | 17 ++++---- .../internal/model/input.go | 41 +++++++++++-------- 5 files changed, 44 insertions(+), 35 deletions(-) delete mode 100644 scanners/subdomain-takeover/internal/messaging/consumer.go diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index 28fb59649b..5d3e3ad110 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -22,12 +22,13 @@ func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { func Classify(input model.Input, matcher BodyFingerprintMatcher) ([]model.Finding, error) { findings := []model.Finding{} - cnameEvidence := ExtractCNAMEEvidence(input) + cnameEvidence := ExtractCNAMEEvidence(input.Results) if cnameEvidence != nil { cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints, matcher) if ShouldEmitCNAME(cnameHit) { findings = append(findings, model.Finding{ Domain: cnameEvidence.Domain, + DomainKey: input.DomainKey, RecordType: model.RecordTypeCNAME, Target: cnameEvidence.Target, Provider: cnameHit.Provider, diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index ba54926bb8..f035d01c76 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -20,26 +20,29 @@ type NSEvidence struct { // Registrar model.RegistrarContext // if/when added } -func ExtractCNAMEEvidence(input model.Input) *CNAMEEvidence { - if input.CnameRecord == nil { +func ExtractCNAMEEvidence(results model.ScanResults) *CNAMEEvidence { + if results.CnameRecord == nil { return nil } + domain := results.Domain + return &CNAMEEvidence{ - Domain: input.Domain, - Target: parseCname(*input.CnameRecord), - NoResolve: input.ResolveChain == nil, + Domain: *domain, + Target: parseCname(*results.CnameRecord), + NoResolve: len(results.ResolveChain) == 0, } } func ExtractNSEvidence(input model.Input) *NSEvidence { - if len(input.NsDelegations.Hosts) == 0 { + nsDelegations := input.Results.NsDelegations + if nsDelegations == nil || len(nsDelegations.Hosts) == 0 { return nil } return &NSEvidence{} } -func ClassifyLameType(nsChecks []model.NsCheck) { +func ClassifyLameType(nsChecks []any) { return } diff --git a/scanners/subdomain-takeover/internal/messaging/consumer.go b/scanners/subdomain-takeover/internal/messaging/consumer.go deleted file mode 100644 index cbb7a43f8b..0000000000 --- a/scanners/subdomain-takeover/internal/messaging/consumer.go +++ /dev/null @@ -1 +0,0 @@ -package messaging diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go index 57dbc83c67..66e98a2f35 100644 --- a/scanners/subdomain-takeover/internal/model/finding.go +++ b/scanners/subdomain-takeover/internal/model/finding.go @@ -8,13 +8,12 @@ const ( ) type Finding struct { - Domain string `json:"domain"` - DomainKey string `json:"domain_key"` - RecordType RecordType `json:"record_type"` - Target string `json:"target"` - Provider string `json:"provider"` - LameType string `json:"lame_type"` - Confidence string `json:"confidence"` - ReasonCode string `json:"reason_code"` - Remediation string `json:"remediation"` + Domain string `json:"domain"` + DomainKey string `json:"domain_key"` + RecordType RecordType `json:"record_type"` + Target string `json:"target"` + Provider string `json:"provider"` + LameType string `json:"lame_type"` + Confidence string `json:"confidence"` + ReasonCode string `json:"reason_code"` } diff --git a/scanners/subdomain-takeover/internal/model/input.go b/scanners/subdomain-takeover/internal/model/input.go index 7bdd179f73..263e74928d 100644 --- a/scanners/subdomain-takeover/internal/model/input.go +++ b/scanners/subdomain-takeover/internal/model/input.go @@ -1,34 +1,30 @@ package model type Input struct { - Domain string `json:"domain"` - BaseDomain string `json:"base_domain"` - ZoneApex string `json:"zone_apex"` - RecordExists bool `json:"record_exists"` - Rcode string `json:"rcode"` - ResolveChain [][]string `json:"resolve_chain"` - CnameRecord *string `json:"cname_record"` - NsRecords NsRecords `json:"ns_records"` - QueryAnswers QueryAnswers `json:"query_res"` - NsDelegations NsDelegations `json:"ns_delegations"` + DomainKey string `json:"domain_key"` + Results ScanResults `json:"results"` +} + +type ScanResults struct { + Domain *string `json:"domain"` + NsDelegations *NsDelegations `json:"ns_delegations"` + RegistrarContext *RegistrarContext `json:"registrar_context"` + ResolveChain [][]string `json:"resolve_chain"` + CnameRecord *string `json:"cname_record"` + NsRecords *NsRecords `json:"ns_records"` } type NsRecords struct { Hostnames []string `json:"hostnames"` Warnings []string `json:"warnings"` -} - -type QueryAnswers struct { - A string `json:"A"` - SOA string `json:"SOA"` - NS string `json:"NS"` - CNAME string `json:"CNAME"` + Error string `json:"error"` } type NsDelegations struct { Hosts []string `json:"ns_hosts"` Checks []NsCheck `json:"ns_checks"` Delegation Delegation `json:"ns_delegation"` + Error string `json:"error"` } type NsCheck struct { @@ -46,3 +42,14 @@ type Delegation struct { LameCount int `json:"lame_count"` LameType string `json:"lame_type"` } + +type RegistrarContext struct { + BaseDomain string `json:"base_domain"` + LookupSuccess bool `json:"lookup_success"` + RDAPURL string `json:"rdap_url"` + RegistrarName string `json:"registrar_name"` + RegistrarID string `json:"registrar_id"` + RDAPNameservers []string `json:"rdap_nameservers"` + DelegationMatchesRDAP *bool `json:"delegation_matches_rdap"` + Error string `json:"error"` +} From 50d653a6728c11ab11f96a22d282b9fbd443ef52 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 15:15:22 -0300 Subject: [PATCH 31/65] add logic to emit nameserver hijacking exploitability --- .../internal/detect/classifier.go | 26 +++-- .../internal/detect/evidence.go | 42 +++++--- .../internal/detect/rules.go | 95 ++++++++++++++++--- 3 files changed, 130 insertions(+), 33 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index 5d3e3ad110..c3883063a0 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -1,6 +1,8 @@ package detect -import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +import ( + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) type Classifier struct { Matcher BodyFingerprintMatcher @@ -38,13 +40,21 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher) ([]model.Findin } } - // nsEvidence := ExtractNSEvidence(input) - // nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) - // if ShouldEmitNSHijack(*nsEvidence, nsHit) { - // findings = append(findings, model.Finding{ - // Domain: input.Domain, - // }) - // } + nsEvidence := ExtractNSEvidence(input.Results) + if nsEvidence != nil { + nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) + if ShouldEmitNSHijack(nsHit) { + findings = append(findings, model.Finding{ + Domain: nsEvidence.Domain, + DomainKey: input.DomainKey, + RecordType: model.RecordTypeNS, + Target: nsHit.Host, + Provider: nsHit.Provider, + ReasonCode: string(nsHit.ReasonCode), + Confidence: ConfidenceForReason(nsHit.ReasonCode), + }) + } + } return findings, nil } diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/evidence.go index f035d01c76..7ea527de08 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/evidence.go @@ -1,6 +1,7 @@ package detect import ( + "slices" "strings" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" @@ -21,34 +22,47 @@ type NSEvidence struct { } func ExtractCNAMEEvidence(results model.ScanResults) *CNAMEEvidence { - if results.CnameRecord == nil { + if results.CnameRecord == nil || results.Domain == nil { return nil } - domain := results.Domain - return &CNAMEEvidence{ - Domain: *domain, + Domain: *results.Domain, Target: parseCname(*results.CnameRecord), NoResolve: len(results.ResolveChain) == 0, } } -func ExtractNSEvidence(input model.Input) *NSEvidence { - nsDelegations := input.Results.NsDelegations - if nsDelegations == nil || len(nsDelegations.Hosts) == 0 { +func ExtractNSEvidence(input model.ScanResults) *NSEvidence { + nsDelegations := input.NsDelegations + if input.Domain == nil || nsDelegations == nil || len(nsDelegations.Hosts) == 0 { return nil } - return &NSEvidence{} -} -func ClassifyLameType(nsChecks []any) { - return + return &NSEvidence{ + Domain: *input.Domain, + NSHosts: parseHostnames(input.NsDelegations.Hosts), + NSDelegations: *input.NsDelegations, + } + } func parseCname(record string) string { - trimmed := strings.Trim(record, ".") - lower := strings.ToLower(trimmed) - recordTokens := strings.Split(lower, " ") + normalRecord := normalizeHost(record) + recordTokens := strings.Split(normalRecord, " ") return recordTokens[len(recordTokens)-1] } + +func parseHostnames(hosts []string) []string { + var normalizedHosts = []string{} + for _, host := range hosts { + normalHost := normalizeHost(host) + normalizedHosts = append(normalizedHosts, normalHost) + } + slices.Sort(normalizedHosts) + return slices.Compact(normalizedHosts) +} + +func normalizeHost(host string) string { + return strings.Trim(strings.ToLower(host), ".") +} diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go index 44626a372b..fe08aabc33 100644 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ b/scanners/subdomain-takeover/internal/detect/rules.go @@ -1,5 +1,7 @@ package detect +import "strings" + type CNAMEHit struct { Matched bool Provider string @@ -8,11 +10,11 @@ type CNAMEHit struct { } type NSHit struct { - Matched bool - Provider string - ReasonCode ReasonCode - ProviderVulnerable bool - RegistrarMismatch bool + Matched bool + Host string + Provider string + ReasonCode ReasonCode + RegistrarMismatch bool } func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { @@ -24,6 +26,7 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider ReasonCode: "", NeedsNX: fp.Nxdomain, } + if hit.NeedsNX { hit.ReasonCode = ReasonCNAMETargetMatchMissingNXDOMAIN if evidence.NoResolve { @@ -38,6 +41,7 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch } } + return &hit } } @@ -45,19 +49,88 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider } func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { - return &NSHit{} + if len(evidence.NSHosts) == 0 || len(fingerprints) == 0 { + return nil + } + + lameType := normalizeLameType(evidence.NSDelegations.Delegation.LameType) + var best *NSHit + + for _, host := range evidence.NSHosts { + for _, fp := range fingerprints { + if fp.ContainsNSHost(host) { + reasonCode := getNSHijackReasonCode(lameType, fp.Status) + hit := &NSHit{ + Matched: isNSMatch(reasonCode), + Host: host, + Provider: fp.Name, + ReasonCode: reasonCode, + } + + if best == nil || nsReasonRank(hit.ReasonCode) > nsReasonRank(best.ReasonCode) { + best = hit + } + } + } + } + + return best } func ShouldEmitCNAME(hit *CNAMEHit) bool { if hit == nil { return false } - if hit.Matched { - return true + return hit.Matched +} + +func ShouldEmitNSHijack(hit *NSHit) bool { + if hit == nil { + return false + } + return hit.Matched +} + +func getNSHijackReasonCode(lameType string, providerStatus NSProviderStatus) ReasonCode { + switch normalizeLameType(lameType) { + case "full": + if isExploitableProviderStatus(providerStatus) { + return ReasonNSFullLameProviderVulnerable + } + return ReasonNSLameProviderUnknown + case "partial": + if isExploitableProviderStatus(providerStatus) { + return ReasonNSPartialLameProviderVulnerable + } + return ReasonNSLameProviderUnknown + default: + return ReasonNSProviderMatchOnly } - return false } -func ShouldEmitNSHijack(evidence NSEvidence, hit *NSHit) bool { - return false +func isNSMatch(reasonCode ReasonCode) bool { + return reasonCode == ReasonNSFullLameProviderVulnerable || reasonCode == ReasonNSPartialLameProviderVulnerable +} + +func isExploitableProviderStatus(status NSProviderStatus) bool { + return status == NSStatusVulnerable || status == NSStatusVulnerableWithPurchase +} + +func normalizeLameType(lameType string) string { + return strings.ToLower(strings.TrimSpace(lameType)) +} + +func nsReasonRank(reasonCode ReasonCode) int { + switch reasonCode { + case ReasonNSFullLameProviderVulnerable: + return 4 + case ReasonNSPartialLameProviderVulnerable: + return 3 + case ReasonNSLameProviderUnknown: + return 2 + case ReasonNSProviderMatchOnly: + return 1 + default: + return 0 + } } From e20f1ea935bc89a3ad1a26e202b0c7b35badbd3c Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 15:20:24 -0300 Subject: [PATCH 32/65] refactor rules.go into separate cname and ns detection logic files --- .../internal/detect/classifier.go | 27 +++- .../internal/detect/cname_evidence.go | 31 ++++ .../internal/detect/cname_rules.go | 46 ++++++ .../internal/detect/fingerprint_source.go | 16 +++ .../detect/{evidence.go => ns_evidence.go} | 32 +---- .../internal/detect/ns_reasoning.go | 47 ++++++ .../internal/detect/ns_rules.go | 45 ++++++ .../internal/detect/rules.go | 136 ------------------ 8 files changed, 210 insertions(+), 170 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/cname_evidence.go create mode 100644 scanners/subdomain-takeover/internal/detect/cname_rules.go create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprint_source.go rename scanners/subdomain-takeover/internal/detect/{evidence.go => ns_evidence.go} (55%) create mode 100644 scanners/subdomain-takeover/internal/detect/ns_reasoning.go create mode 100644 scanners/subdomain-takeover/internal/detect/ns_rules.go delete mode 100644 scanners/subdomain-takeover/internal/detect/rules.go diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index c3883063a0..d3853749fd 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -6,10 +6,19 @@ import ( type Classifier struct { Matcher BodyFingerprintMatcher + Source FingerprintSource } func NewClassifier(matcher BodyFingerprintMatcher) *Classifier { - return &Classifier{Matcher: matcher} + return &Classifier{Matcher: matcher, Source: GlobalFingerprintSource{}} +} + +func NewClassifierWithSource(matcher BodyFingerprintMatcher, source FingerprintSource) *Classifier { + if source == nil { + source = GlobalFingerprintSource{} + } + + return &Classifier{Matcher: matcher, Source: source} } func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { @@ -18,15 +27,23 @@ func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { matcher = NewNoopBodyFingerprintMatcher() } - return Classify(input, matcher) + source := c.Source + if source == nil { + source = GlobalFingerprintSource{} + } + + return Classify(input, matcher, source) } -func Classify(input model.Input, matcher BodyFingerprintMatcher) ([]model.Finding, error) { +func Classify(input model.Input, matcher BodyFingerprintMatcher, source FingerprintSource) ([]model.Finding, error) { findings := []model.Finding{} + cnameFingerprints := source.CNAME() + nsFingerprints := source.NS() + cnameEvidence := ExtractCNAMEEvidence(input.Results) if cnameEvidence != nil { - cnameHit := MatchCNAMEFingerprints(*cnameEvidence, CNAMEProviderFingerprints, matcher) + cnameHit := MatchCNAMEFingerprints(*cnameEvidence, cnameFingerprints, matcher) if ShouldEmitCNAME(cnameHit) { findings = append(findings, model.Finding{ Domain: cnameEvidence.Domain, @@ -42,7 +59,7 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher) ([]model.Findin nsEvidence := ExtractNSEvidence(input.Results) if nsEvidence != nil { - nsHit := MatchNSProviderRules(*nsEvidence, NSProviderFingerprints) + nsHit := MatchNSProviderRules(*nsEvidence, nsFingerprints) if ShouldEmitNSHijack(nsHit) { findings = append(findings, model.Finding{ Domain: nsEvidence.Domain, diff --git a/scanners/subdomain-takeover/internal/detect/cname_evidence.go b/scanners/subdomain-takeover/internal/detect/cname_evidence.go new file mode 100644 index 0000000000..4466dcd9fc --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/cname_evidence.go @@ -0,0 +1,31 @@ +package detect + +import ( + "strings" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +type CNAMEEvidence struct { + Domain string + Target string + NoResolve bool +} + +func ExtractCNAMEEvidence(results model.ScanResults) *CNAMEEvidence { + if results.CnameRecord == nil || results.Domain == nil { + return nil + } + + return &CNAMEEvidence{ + Domain: *results.Domain, + Target: parseCname(*results.CnameRecord), + NoResolve: len(results.ResolveChain) == 0, + } +} + +func parseCname(record string) string { + normalRecord := normalizeHost(record) + recordTokens := strings.Split(normalRecord, " ") + return recordTokens[len(recordTokens)-1] +} diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules.go b/scanners/subdomain-takeover/internal/detect/cname_rules.go new file mode 100644 index 0000000000..e69c5cddb0 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/cname_rules.go @@ -0,0 +1,46 @@ +package detect + +type CNAMEHit struct { + Matched bool + Provider string + ReasonCode ReasonCode + NeedsNX bool +} + +func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { + for _, fp := range fingerprints { + if fp.ContainsTarget(evidence.Target) { + hit := CNAMEHit{ + Matched: false, + Provider: fp.Name, + ReasonCode: "", + NeedsNX: fp.Nxdomain, + } + + if hit.NeedsNX { + hit.ReasonCode = ReasonCNAMETargetMatchMissingNXDOMAIN + if evidence.NoResolve { + hit.Matched = true + hit.ReasonCode = ReasonCNAMEDanglingNXDOMAIN + } + } else { + hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP + mode := normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + if matcher != nil && matcher.Contains(evidence.Domain, fp.Fingerprint, mode) { + hit.Matched = true + hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch + } + } + + return &hit + } + } + return nil +} + +func ShouldEmitCNAME(hit *CNAMEHit) bool { + if hit == nil { + return false + } + return hit.Matched +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_source.go b/scanners/subdomain-takeover/internal/detect/fingerprint_source.go new file mode 100644 index 0000000000..2ecc9981f8 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_source.go @@ -0,0 +1,16 @@ +package detect + +type FingerprintSource interface { + CNAME() []CNAMEProviderFingerprint + NS() []NSProviderFingerprint +} + +type GlobalFingerprintSource struct{} + +func (GlobalFingerprintSource) CNAME() []CNAMEProviderFingerprint { + return CNAMEProviderFingerprints +} + +func (GlobalFingerprintSource) NS() []NSProviderFingerprint { + return NSProviderFingerprints +} diff --git a/scanners/subdomain-takeover/internal/detect/evidence.go b/scanners/subdomain-takeover/internal/detect/ns_evidence.go similarity index 55% rename from scanners/subdomain-takeover/internal/detect/evidence.go rename to scanners/subdomain-takeover/internal/detect/ns_evidence.go index 7ea527de08..78ddb2f428 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence.go +++ b/scanners/subdomain-takeover/internal/detect/ns_evidence.go @@ -7,13 +7,6 @@ import ( "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) -// inputs from evidence.go -type CNAMEEvidence struct { - Domain string - Target string - NoResolve bool -} - type NSEvidence struct { Domain string NSHosts []string @@ -21,18 +14,6 @@ type NSEvidence struct { // Registrar model.RegistrarContext // if/when added } -func ExtractCNAMEEvidence(results model.ScanResults) *CNAMEEvidence { - if results.CnameRecord == nil || results.Domain == nil { - return nil - } - - return &CNAMEEvidence{ - Domain: *results.Domain, - Target: parseCname(*results.CnameRecord), - NoResolve: len(results.ResolveChain) == 0, - } -} - func ExtractNSEvidence(input model.ScanResults) *NSEvidence { nsDelegations := input.NsDelegations if input.Domain == nil || nsDelegations == nil || len(nsDelegations.Hosts) == 0 { @@ -44,21 +25,14 @@ func ExtractNSEvidence(input model.ScanResults) *NSEvidence { NSHosts: parseHostnames(input.NsDelegations.Hosts), NSDelegations: *input.NsDelegations, } - -} - -func parseCname(record string) string { - normalRecord := normalizeHost(record) - recordTokens := strings.Split(normalRecord, " ") - return recordTokens[len(recordTokens)-1] } func parseHostnames(hosts []string) []string { - var normalizedHosts = []string{} + normalizedHosts := make([]string, 0, len(hosts)) for _, host := range hosts { - normalHost := normalizeHost(host) - normalizedHosts = append(normalizedHosts, normalHost) + normalizedHosts = append(normalizedHosts, normalizeHost(host)) } + slices.Sort(normalizedHosts) return slices.Compact(normalizedHosts) } diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go new file mode 100644 index 0000000000..953cee5580 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go @@ -0,0 +1,47 @@ +package detect + +import "strings" + +func getNSHijackReasonCode(lameType string, providerStatus NSProviderStatus) ReasonCode { + switch normalizeLameType(lameType) { + case "full": + if isExploitableProviderStatus(providerStatus) { + return ReasonNSFullLameProviderVulnerable + } + return ReasonNSLameProviderUnknown + case "partial": + if isExploitableProviderStatus(providerStatus) { + return ReasonNSPartialLameProviderVulnerable + } + return ReasonNSLameProviderUnknown + default: + return ReasonNSProviderMatchOnly + } +} + +func isNSMatch(reasonCode ReasonCode) bool { + return reasonCode == ReasonNSFullLameProviderVulnerable || reasonCode == ReasonNSPartialLameProviderVulnerable +} + +func isExploitableProviderStatus(status NSProviderStatus) bool { + return status == NSStatusVulnerable || status == NSStatusVulnerableWithPurchase +} + +func normalizeLameType(lameType string) string { + return strings.ToLower(strings.TrimSpace(lameType)) +} + +func nsReasonRank(reasonCode ReasonCode) int { + switch reasonCode { + case ReasonNSFullLameProviderVulnerable: + return 4 + case ReasonNSPartialLameProviderVulnerable: + return 3 + case ReasonNSLameProviderUnknown: + return 2 + case ReasonNSProviderMatchOnly: + return 1 + default: + return 0 + } +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules.go b/scanners/subdomain-takeover/internal/detect/ns_rules.go new file mode 100644 index 0000000000..ff993ef28b --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_rules.go @@ -0,0 +1,45 @@ +package detect + +type NSHit struct { + Matched bool + Host string + Provider string + ReasonCode ReasonCode + RegistrarMismatch bool +} + +func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { + if len(evidence.NSHosts) == 0 || len(fingerprints) == 0 { + return nil + } + + lameType := normalizeLameType(evidence.NSDelegations.Delegation.LameType) + var best *NSHit + + for _, host := range evidence.NSHosts { + for _, fp := range fingerprints { + if fp.ContainsNSHost(host) { + reasonCode := getNSHijackReasonCode(lameType, fp.Status) + hit := &NSHit{ + Matched: isNSMatch(reasonCode), + Host: host, + Provider: fp.Name, + ReasonCode: reasonCode, + } + + if best == nil || nsReasonRank(hit.ReasonCode) > nsReasonRank(best.ReasonCode) { + best = hit + } + } + } + } + + return best +} + +func ShouldEmitNSHijack(hit *NSHit) bool { + if hit == nil { + return false + } + return hit.Matched +} diff --git a/scanners/subdomain-takeover/internal/detect/rules.go b/scanners/subdomain-takeover/internal/detect/rules.go deleted file mode 100644 index fe08aabc33..0000000000 --- a/scanners/subdomain-takeover/internal/detect/rules.go +++ /dev/null @@ -1,136 +0,0 @@ -package detect - -import "strings" - -type CNAMEHit struct { - Matched bool - Provider string - ReasonCode ReasonCode - NeedsNX bool -} - -type NSHit struct { - Matched bool - Host string - Provider string - ReasonCode ReasonCode - RegistrarMismatch bool -} - -func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { - for _, fp := range fingerprints { - if fp.ContainsTarget(evidence.Target) { - hit := CNAMEHit{ - Matched: false, - Provider: fp.Name, - ReasonCode: "", - NeedsNX: fp.Nxdomain, - } - - if hit.NeedsNX { - hit.ReasonCode = ReasonCNAMETargetMatchMissingNXDOMAIN - if evidence.NoResolve { - hit.Matched = true - hit.ReasonCode = ReasonCNAMEDanglingNXDOMAIN - } - } else { - hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP - mode := normalizeFingerprintMode(fp.Mode, fp.Fingerprint) - if matcher != nil && matcher.Contains(evidence.Domain, fp.Fingerprint, mode) { - hit.Matched = true - hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch - } - } - - return &hit - } - } - return nil -} - -func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { - if len(evidence.NSHosts) == 0 || len(fingerprints) == 0 { - return nil - } - - lameType := normalizeLameType(evidence.NSDelegations.Delegation.LameType) - var best *NSHit - - for _, host := range evidence.NSHosts { - for _, fp := range fingerprints { - if fp.ContainsNSHost(host) { - reasonCode := getNSHijackReasonCode(lameType, fp.Status) - hit := &NSHit{ - Matched: isNSMatch(reasonCode), - Host: host, - Provider: fp.Name, - ReasonCode: reasonCode, - } - - if best == nil || nsReasonRank(hit.ReasonCode) > nsReasonRank(best.ReasonCode) { - best = hit - } - } - } - } - - return best -} - -func ShouldEmitCNAME(hit *CNAMEHit) bool { - if hit == nil { - return false - } - return hit.Matched -} - -func ShouldEmitNSHijack(hit *NSHit) bool { - if hit == nil { - return false - } - return hit.Matched -} - -func getNSHijackReasonCode(lameType string, providerStatus NSProviderStatus) ReasonCode { - switch normalizeLameType(lameType) { - case "full": - if isExploitableProviderStatus(providerStatus) { - return ReasonNSFullLameProviderVulnerable - } - return ReasonNSLameProviderUnknown - case "partial": - if isExploitableProviderStatus(providerStatus) { - return ReasonNSPartialLameProviderVulnerable - } - return ReasonNSLameProviderUnknown - default: - return ReasonNSProviderMatchOnly - } -} - -func isNSMatch(reasonCode ReasonCode) bool { - return reasonCode == ReasonNSFullLameProviderVulnerable || reasonCode == ReasonNSPartialLameProviderVulnerable -} - -func isExploitableProviderStatus(status NSProviderStatus) bool { - return status == NSStatusVulnerable || status == NSStatusVulnerableWithPurchase -} - -func normalizeLameType(lameType string) string { - return strings.ToLower(strings.TrimSpace(lameType)) -} - -func nsReasonRank(reasonCode ReasonCode) int { - switch reasonCode { - case ReasonNSFullLameProviderVulnerable: - return 4 - case ReasonNSPartialLameProviderVulnerable: - return 3 - case ReasonNSLameProviderUnknown: - return 2 - case ReasonNSProviderMatchOnly: - return 1 - default: - return 0 - } -} From dc360ef065a24ce47a03e86d1e42b7845f26ccd0 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 16:44:06 -0300 Subject: [PATCH 33/65] add debug logging --- .../subdomain-takeover/cmd/service/main.go | 17 ++++-- .../subdomain-takeover/internal/app/runner.go | 14 ++++- .../subdomain-takeover/internal/app/worker.go | 14 ++++- .../internal/bootstrap/nats.go | 8 ++- .../internal/detect/classifier.go | 60 +++++++++++++++++-- .../internal/detect/cname_rules.go | 31 ++++++++++ .../internal/detect/fingerprint_loader.go | 12 +++- .../internal/detect/logger.go | 9 +++ .../internal/detect/ns_rules.go | 28 ++++++++- .../internal/messaging/publisher.go | 28 ++++++++- 10 files changed, 204 insertions(+), 17 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/logger.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 0f588ec30d..78a2500bff 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -24,8 +24,17 @@ func main() { } logger = bootstrap.NewLogger(cfg.LogLevel) - - if err := detect.LoadFingerprints(); err != nil { + logger.Info(). + Str("nats_url", cfg.NATSURL). + Str("stream", cfg.NATSStream). + Str("subject_in", cfg.SubjectIn). + Str("subject_out", cfg.SubjectOut). + Str("durable", cfg.DurableName). + Int("worker_count", cfg.WorkerCount). + Str("log_level", cfg.LogLevel.String()). + Msg("service configuration loaded") + + if err := detect.LoadFingerprints(logger); err != nil { logger.Fatal().Err(err).Msg("failed to load fingerprints") } @@ -36,7 +45,7 @@ func main() { signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) defer signal.Stop(sig) - runtimeDeps, err := bootstrap.NewRuntimeDeps(ctx, cfg) + runtimeDeps, err := bootstrap.NewRuntimeDeps(ctx, cfg, logger) if err != nil { logger.Fatal().Err(err).Msg("failed to initialize runtime dependencies") } @@ -45,7 +54,7 @@ func main() { pub := messaging.NewPublisher(logger, runtimeDeps.JS, cfg.SubjectOut) matcher := detect.NewHTTPBodyFingerprintMatcher(5 * time.Second) - classifier := detect.NewClassifier(matcher) + classifier := detect.NewClassifier(matcher).WithLogger(logger) worker := app.NewWorker(logger, pub, classifier) go func() { diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index 6d7ee52e8d..b8c0121f87 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -26,11 +26,14 @@ type MessageHandler interface { func Run(ctx context.Context, deps RunnerDeps) { logger := deps.Logger iter := deps.Iter + nextErrCount := 0 if deps.WorkerCount < 1 { deps.WorkerCount = 1 } + logger.Info().Int("worker_count", deps.WorkerCount).Msg("runner started") + var wg sync.WaitGroup sem := make(chan struct{}, deps.WorkerCount) @@ -50,11 +53,18 @@ Loop: msg, err := iter.Next() if err != nil { if ctx.Err() != nil { + logger.Info().Msg("runner stopping: context canceled") break Loop } - logger.Debug().Err(err).Msg("next returned, continuing") + nextErrCount++ + if nextErrCount%10 == 0 { + logger.Warn().Err(err).Int("consecutive_next_errors", nextErrCount).Msg("iterator next repeatedly failed") + } else { + logger.Debug().Err(err).Int("consecutive_next_errors", nextErrCount).Msg("next returned, continuing") + } continue } + nextErrCount = 0 select { case sem <- struct{}{}: @@ -80,6 +90,6 @@ Loop: case <-waitDone: logger.Info().Msg("all workers drained") case <-time.After(30 * time.Second): - logger.Info().Msg("drain timeout") + logger.Warn().Msg("drain timeout") } } diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index e282fab952..eca59f91bc 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -30,7 +30,11 @@ func NewWorker(logger zerolog.Logger, publisher FindingPublisher, classifier Sca } func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { - log := w.logger.With().Str("component", "worker").Logger() + log := w.logger.With(). + Str("component", "worker"). + Str("subject", msg.Subject()). + Int("msg_size", len(msg.Data())). + Logger() scan, err := decodeScan(msg.Data()) if err != nil { @@ -39,6 +43,8 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { return err } + log = log.With().Str("domain_key", scan.DomainKey).Logger() + findings, err := w.classifier.Classify(scan) if err != nil { log.Err(err).Msg("classify error") @@ -46,6 +52,10 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { return err } + if len(findings) == 0 { + log.Debug().Msg("classification produced no findings") + } + for _, finding := range findings { err = w.publisher.Publish(ctx, finding) if err != nil { @@ -60,6 +70,8 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { return err } + log.Debug().Int("findings_count", len(findings)).Msg("message acknowledged") + return nil } diff --git a/scanners/subdomain-takeover/internal/bootstrap/nats.go b/scanners/subdomain-takeover/internal/bootstrap/nats.go index 51edb6b060..ea2fc49e21 100644 --- a/scanners/subdomain-takeover/internal/bootstrap/nats.go +++ b/scanners/subdomain-takeover/internal/bootstrap/nats.go @@ -7,6 +7,7 @@ import ( "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" ) type RuntimeDeps struct { @@ -15,18 +16,21 @@ type RuntimeDeps struct { JS jetstream.JetStream } -func NewRuntimeDeps(ctx context.Context, cfg *config.Config) (*RuntimeDeps, error) { +func NewRuntimeDeps(ctx context.Context, cfg *config.Config, logger zerolog.Logger) (*RuntimeDeps, error) { + logger.Debug().Str("nats_url", cfg.NATSURL).Msg("connecting to nats") nc, err := nats.Connect(cfg.NATSURL) if err != nil { return nil, err } + logger.Debug().Msg("creating jetstream client") js, err := jetstream.New(nc) if err != nil { nc.Close() return nil, err } + logger.Debug().Str("stream", cfg.NATSStream).Strs("subjects", []string{cfg.SubjectIn, cfg.SubjectOut}).Msg("creating or updating stream") stream, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ Name: cfg.NATSStream, Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, @@ -36,6 +40,7 @@ func NewRuntimeDeps(ctx context.Context, cfg *config.Config) (*RuntimeDeps, erro return nil, err } + logger.Debug().Str("durable", cfg.DurableName).Msg("creating or updating consumer") cons, err := stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ Durable: cfg.DurableName, AckPolicy: jetstream.AckExplicitPolicy, @@ -45,6 +50,7 @@ func NewRuntimeDeps(ctx context.Context, cfg *config.Config) (*RuntimeDeps, erro return nil, err } + logger.Debug().Msg("creating pull message iterator") iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) if err != nil { nc.Close() diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index d3853749fd..ffd1d527c6 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -2,15 +2,17 @@ package detect import ( "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/rs/zerolog" ) type Classifier struct { Matcher BodyFingerprintMatcher Source FingerprintSource + logger zerolog.Logger } func NewClassifier(matcher BodyFingerprintMatcher) *Classifier { - return &Classifier{Matcher: matcher, Source: GlobalFingerprintSource{}} + return &Classifier{Matcher: matcher, Source: GlobalFingerprintSource{}, logger: zerolog.Nop()} } func NewClassifierWithSource(matcher BodyFingerprintMatcher, source FingerprintSource) *Classifier { @@ -18,7 +20,13 @@ func NewClassifierWithSource(matcher BodyFingerprintMatcher, source FingerprintS source = GlobalFingerprintSource{} } - return &Classifier{Matcher: matcher, Source: source} + return &Classifier{Matcher: matcher, Source: source, logger: zerolog.Nop()} +} + +func (c *Classifier) WithLogger(logger zerolog.Logger) *Classifier { + c.logger = logger.With().Str("component", "classifier").Logger() + SetLogger(logger) + return c } func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { @@ -32,10 +40,15 @@ func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { source = GlobalFingerprintSource{} } - return Classify(input, matcher, source) + logger := c.logger + if logger.GetLevel() == zerolog.NoLevel { + logger = zerolog.Nop() + } + + return Classify(input, matcher, source, logger) } -func Classify(input model.Input, matcher BodyFingerprintMatcher, source FingerprintSource) ([]model.Finding, error) { +func Classify(input model.Input, matcher BodyFingerprintMatcher, source FingerprintSource, logger zerolog.Logger) ([]model.Finding, error) { findings := []model.Finding{} cnameFingerprints := source.CNAME() @@ -43,8 +56,15 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, source Fingerpr cnameEvidence := ExtractCNAMEEvidence(input.Results) if cnameEvidence != nil { + logger.Debug().Str("domain_key", input.DomainKey).Str("domain", cnameEvidence.Domain).Msg("cname evidence extracted") cnameHit := MatchCNAMEFingerprints(*cnameEvidence, cnameFingerprints, matcher) if ShouldEmitCNAME(cnameHit) { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", cnameEvidence.Domain). + Str("provider", cnameHit.Provider). + Str("reason_code", string(cnameHit.ReasonCode)). + Msg("emitting cname finding") findings = append(findings, model.Finding{ Domain: cnameEvidence.Domain, DomainKey: input.DomainKey, @@ -55,12 +75,34 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, source Fingerpr Confidence: ConfidenceForReason(cnameHit.ReasonCode), }) } + if cnameHit != nil && !ShouldEmitCNAME(cnameHit) { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", cnameEvidence.Domain). + Str("provider", cnameHit.Provider). + Str("reason_code", string(cnameHit.ReasonCode)). + Msg("cname finding suppressed") + } + } else { + logger.Debug().Str("domain_key", input.DomainKey).Msg("no cname evidence") } nsEvidence := ExtractNSEvidence(input.Results) if nsEvidence != nil { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", nsEvidence.Domain). + Int("ns_hosts", len(nsEvidence.NSHosts)). + Msg("ns evidence extracted") nsHit := MatchNSProviderRules(*nsEvidence, nsFingerprints) if ShouldEmitNSHijack(nsHit) { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", nsEvidence.Domain). + Str("provider", nsHit.Provider). + Str("target", nsHit.Host). + Str("reason_code", string(nsHit.ReasonCode)). + Msg("emitting ns finding") findings = append(findings, model.Finding{ Domain: nsEvidence.Domain, DomainKey: input.DomainKey, @@ -70,7 +112,17 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, source Fingerpr ReasonCode: string(nsHit.ReasonCode), Confidence: ConfidenceForReason(nsHit.ReasonCode), }) + } else if nsHit != nil { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", nsEvidence.Domain). + Str("provider", nsHit.Provider). + Str("target", nsHit.Host). + Str("reason_code", string(nsHit.ReasonCode)). + Msg("ns finding suppressed") } + } else { + logger.Debug().Str("domain_key", input.DomainKey).Msg("no ns evidence") } return findings, nil diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules.go b/scanners/subdomain-takeover/internal/detect/cname_rules.go index e69c5cddb0..b693a4cd70 100644 --- a/scanners/subdomain-takeover/internal/detect/cname_rules.go +++ b/scanners/subdomain-takeover/internal/detect/cname_rules.go @@ -10,6 +10,13 @@ type CNAMEHit struct { func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { for _, fp := range fingerprints { if fp.ContainsTarget(evidence.Target) { + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("target", evidence.Target). + Str("provider", fp.Name). + Bool("needs_nx", fp.Nxdomain). + Msg("cname provider target matched") + hit := CNAMEHit{ Matched: false, Provider: fp.Name, @@ -26,12 +33,36 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider } else { hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP mode := normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("fingerprint_mode", string(mode)). + Msg("checking cname body fingerprint") if matcher != nil && matcher.Contains(evidence.Domain, fp.Fingerprint, mode) { hit.Matched = true hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("reason_code", string(hit.ReasonCode)). + Msg("cname body fingerprint matched") + } else { + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("reason_code", string(hit.ReasonCode)). + Msg("cname finding suppressed: body fingerprint missing") } } + if hit.NeedsNX && !hit.Matched { + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("reason_code", string(hit.ReasonCode)). + Msg("cname finding suppressed: required nxdomain signal missing") + } + return &hit } } diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go b/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go index 694bf833aa..da2101e51a 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go @@ -6,6 +6,8 @@ import ( "fmt" "strings" "sync" + + "github.com/rs/zerolog" ) //go:embed data/*.json @@ -16,33 +18,41 @@ var ( loadFingerprintsErr error ) -func LoadFingerprints() error { +func LoadFingerprints(logger zerolog.Logger) error { + log := logger.With().Str("component", "fingerprint_loader").Logger() + loadFingerprintsOnce.Do(func() { var cname []CNAMEProviderFingerprint var ns []NSProviderFingerprint if err := loadJSON("data/cname_fingerprints.json", &cname); err != nil { + log.Error().Err(err).Str("dataset", "cname_fingerprints").Msg("failed to load fingerprint dataset") loadFingerprintsErr = err return } if err := loadJSON("data/ns_fingerprints.json", &ns); err != nil { + log.Error().Err(err).Str("dataset", "ns_fingerprints").Msg("failed to load fingerprint dataset") loadFingerprintsErr = err return } if err := validateCNAMEFingerprints(cname); err != nil { + log.Error().Err(err).Str("dataset", "cname_fingerprints").Msg("invalid fingerprint dataset") loadFingerprintsErr = err return } if err := validateNSFingerprints(ns); err != nil { + log.Error().Err(err).Str("dataset", "ns_fingerprints").Msg("invalid fingerprint dataset") loadFingerprintsErr = err return } CNAMEProviderFingerprints = cname NSProviderFingerprints = ns + + log.Info().Int("cname_fingerprints", len(cname)).Int("ns_fingerprints", len(ns)).Msg("fingerprint datasets loaded") }) return loadFingerprintsErr diff --git a/scanners/subdomain-takeover/internal/detect/logger.go b/scanners/subdomain-takeover/internal/detect/logger.go new file mode 100644 index 0000000000..c3d4a30293 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/logger.go @@ -0,0 +1,9 @@ +package detect + +import "github.com/rs/zerolog" + +var detectLogger = zerolog.Nop() + +func SetLogger(logger zerolog.Logger) { + detectLogger = logger.With().Str("component", "detect").Logger() +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules.go b/scanners/subdomain-takeover/internal/detect/ns_rules.go index ff993ef28b..5485e9cc95 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_rules.go +++ b/scanners/subdomain-takeover/internal/detect/ns_rules.go @@ -5,11 +5,11 @@ type NSHit struct { Host string Provider string ReasonCode ReasonCode - RegistrarMismatch bool } func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { if len(evidence.NSHosts) == 0 || len(fingerprints) == 0 { + detectLogger.Debug().Int("ns_hosts", len(evidence.NSHosts)).Int("fingerprints", len(fingerprints)).Msg("skipping ns matching due to insufficient inputs") return nil } @@ -20,6 +20,7 @@ func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerpr for _, fp := range fingerprints { if fp.ContainsNSHost(host) { reasonCode := getNSHijackReasonCode(lameType, fp.Status) + rank := nsReasonRank(reasonCode) hit := &NSHit{ Matched: isNSMatch(reasonCode), Host: host, @@ -27,6 +28,17 @@ func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerpr ReasonCode: reasonCode, } + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("host", host). + Str("provider", fp.Name). + Str("provider_status", string(fp.Status)). + Str("lame_type", lameType). + Str("reason_code", string(reasonCode)). + Int("rank", rank). + Bool("emittable", hit.Matched). + Msg("ns candidate evaluated") + if best == nil || nsReasonRank(hit.ReasonCode) > nsReasonRank(best.ReasonCode) { best = hit } @@ -34,6 +46,20 @@ func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerpr } } + if best == nil { + detectLogger.Debug().Str("domain", evidence.Domain).Msg("no ns provider match") + return nil + } + + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("host", best.Host). + Str("provider", best.Provider). + Str("reason_code", string(best.ReasonCode)). + Int("rank", nsReasonRank(best.ReasonCode)). + Bool("emittable", best.Matched). + Msg("ns best candidate selected") + return best } diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go index 5b5585a75f..5626f45983 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -22,15 +22,37 @@ func NewPublisher(logger zerolog.Logger, js jetstream.JetStream, subject string) func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { payload, err := json.Marshal(finding) if err != nil { - p.logger.Error().Err(err).Str("domain", finding.Domain).Msg("marshal finding failed") + p.logger.Error(). + Err(err). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Str("reason_code", finding.ReasonCode). + Str("confidence", finding.Confidence). + Msg("marshal finding failed") return err } if _, err := p.js.Publish(ctx, p.subject, payload); err != nil { - p.logger.Error().Err(err).Str("domain", finding.Domain).Str("subject", p.subject).Msg("publish failed") + p.logger.Error(). + Err(err). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Str("reason_code", finding.ReasonCode). + Str("confidence", finding.Confidence). + Str("subject", p.subject). + Msg("publish failed") return err } - p.logger.Debug().Str("domain", finding.Domain).Str("subject", p.subject).Msg("finding published") + p.logger.Debug(). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Str("reason_code", finding.ReasonCode). + Str("confidence", finding.Confidence). + Str("subject", p.subject). + Msg("finding published") return nil } From 92c33569e8990ce92c682a405d4caca5c16b8393 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 16:44:11 -0300 Subject: [PATCH 34/65] update README.md --- scanners/subdomain-takeover/README.md | 54 ++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md index 640b246776..7e28388320 100644 --- a/scanners/subdomain-takeover/README.md +++ b/scanners/subdomain-takeover/README.md @@ -5,16 +5,27 @@ Go microservice that consumes DNS scan results and emits normalized subdomain ta ## What it does 1. Consumes from `scans.dns_scanner_results` (JetStream stream: `SCANS`) -2. Evaluates passive CNAME takeover risk using provider fingerprints +2. Evaluates passive CNAME and NS takeover risk using provider fingerprints 3. Assigns confidence (`suspected`, `probable`, `confirmed`) 4. Publishes findings to `scans.findings.upsert` ## Current detection scope -- Implemented: CNAME-based passive detection -- Not implemented yet: NS-based (Sitting Duck) detection +Implemented today: -The service currently focuses on passive CNAME classification only. NS-related types and reason codes exist in the codebase for upcoming work, but NS findings are not emitted yet. +- CNAME passive detection +- NS passive detection (vulnerable-only emission policy) +- Deterministic NS candidate selection when multiple providers/hosts match +- Nil-safe evidence extraction for CNAME and NS +- Classifier refactor to support injected fingerprint sources (`FingerprintSource`) + +Current NS behavior: + +- NS provider/host matches are evaluated and ranked +- Findings are emitted only for exploitable NS outcomes: + - `NS_FULL_LAME_PROVIDER_VULNERABLE` + - `NS_PARTIAL_LAME_PROVIDER_VULNERABLE` +- Non-exploitable NS matches (`NS_LAME_PROVIDER_UNKNOWN`, `NS_PROVIDER_MATCH_ONLY`) are classified but not emitted ## Passive-only detection policy @@ -28,7 +39,7 @@ As a result, confidence is conservative and evidence-driven: ## Confidence model (current) -Current production behavior for CNAME findings: +### CNAME - `suspected` - provider target matched, but required passive signal is missing @@ -38,6 +49,28 @@ Current production behavior for CNAME findings: - `confirmed` - reserved; not currently emitted by CNAME rules +### NS + +- `confirmed` + - full lame delegation + exploitable provider status +- `probable` + - partial lame delegation + exploitable provider status +- `suspected` + - reason codes exist for weaker NS states, but these are currently not emitted + +## Internal package layout + +Detection logic was split into focused files to reduce coupling: + +- `internal/detect/cname_rules.go` +- `internal/detect/ns_rules.go` +- `internal/detect/ns_reasoning.go` +- `internal/detect/cname_evidence.go` +- `internal/detect/ns_evidence.go` +- `internal/detect/fingerprint_source.go` + +`Classifier` now supports dependency injection via `NewClassifierWithSource(...)`, while `NewClassifier(...)` defaults to global loaded fingerprints. + ## Fingerprint data Provider fingerprint data is stored in: @@ -88,7 +121,7 @@ The worker uses explicit JetStream ack semantics: Publish a test request: ```bash -nats pub scans.dns_scanner_results '{"domain":"example.gc.ca","domain_key":"12345","base_domain":"example.gc.ca","zone_apex":"example.gc.ca","record_exists":true,"rcode":"NOERROR","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_records":{"hostnames":["ns1.example-dns-provider.net"],"warnings":[]}}' +nats pub scans.dns_scanner_results '{"domain_key":"12345","results":{"domain":"example.gc.ca","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_delegations":{"ns_hosts":["ns1.example-dns-provider.net"],"ns_checks":[],"ns_delegation":{"total_ns":1,"authoritative_ok":0,"lame_count":1,"lame_type":"full"},"error":""}}}' ``` Watch findings: @@ -96,3 +129,12 @@ Watch findings: ```bash nats sub "scans.findings.upsert" ``` + +## Next steps / nice-to-haves + +1. Add table-driven tests for NS reason mapping, ranking, and emission gating. +2. Add classifier integration tests using injected `FingerprintSource`. +3. Document and implement explicit policy for `edge_case` and `registration_closed` NS provider statuses. +4. Add structured debug logs for NS matching decisions (host, provider, status, reason). +5. Add registrar-context mismatch handling (`RegistrarMismatch`) and confidence policy. +6. Add metrics (counts by reason code, emitted vs suppressed) for production observability. From ca11886b0d250b3fb40a171608a04dda9e869aa2 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 17:00:44 -0300 Subject: [PATCH 35/65] add detect logic tests --- .../internal/detect/body_matcher_test.go | 63 +++++++++ .../internal/detect/classifier_test.go | 132 ++++++++++++++++++ .../internal/detect/cname_rules_test.go | 102 ++++++++++++++ .../internal/detect/confidence_test.go | 26 ++++ .../internal/detect/evidence_test.go | 128 +++++++++++++++++ .../detect/fingerprint_loader_test.go | 47 +++++++ .../detect/fingerprint_source_test.go | 23 +++ .../internal/detect/fingerprints_test.go | 47 +++++++ .../internal/detect/logger_test.go | 15 ++ .../internal/detect/ns_reasoning_test.go | 62 ++++++++ .../internal/detect/ns_rules_test.go | 110 +++++++++++++++ .../internal/detect/test_helpers_test.go | 32 +++++ 12 files changed, 787 insertions(+) create mode 100644 scanners/subdomain-takeover/internal/detect/body_matcher_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/classifier_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/cname_rules_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/confidence_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/evidence_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprint_loader_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprints_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/logger_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/ns_rules_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/test_helpers_test.go diff --git a/scanners/subdomain-takeover/internal/detect/body_matcher_test.go b/scanners/subdomain-takeover/internal/detect/body_matcher_test.go new file mode 100644 index 0000000000..a045a83a48 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/body_matcher_test.go @@ -0,0 +1,63 @@ +package detect + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestNoopBodyFingerprintMatcher(t *testing.T) { + matcher := NewNoopBodyFingerprintMatcher() + if matcher.Contains("example.ca", "anything", FingerprintModeLiteral) { + t.Fatal("noop matcher should never match") + } +} + +func TestHTTPBodyFingerprintMatcher(t *testing.T) { + t.Run("returns false when matcher/client nil", func(t *testing.T) { + var matcher *HTTPBodyFingerprintMatcher + if matcher.Contains("example.ca", "x", FingerprintModeLiteral) { + t.Fatal("expected false for nil matcher") + } + + matcher = &HTTPBodyFingerprintMatcher{} + if matcher.Contains("example.ca", "x", FingerprintModeLiteral) { + t.Fatal("expected false for nil client") + } + }) + + t.Run("literal and regex matching", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("Hello from sample app. Error Code: 503")) + })) + t.Cleanup(server.Close) + + domain := strings.TrimPrefix(server.URL, "http://") + matcher := NewHTTPBodyFingerprintMatcher(2 * time.Second) + + if !matcher.Contains(domain, "sample app", FingerprintModeLiteral) { + t.Fatal("expected literal substring match") + } + if !matcher.Contains(domain, `Error Code: \d+`, FingerprintModeRegex) { + t.Fatal("expected regex match") + } + if matcher.Contains(domain, "does-not-exist", FingerprintModeLiteral) { + t.Fatal("did not expect missing literal to match") + } + }) + + t.Run("invalid regex returns false", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("anything")) + })) + t.Cleanup(server.Close) + + domain := strings.TrimPrefix(server.URL, "http://") + matcher := NewHTTPBodyFingerprintMatcher(2 * time.Second) + if matcher.Contains(domain, "(", FingerprintModeRegex) { + t.Fatal("expected false for invalid regex") + } + }) +} diff --git a/scanners/subdomain-takeover/internal/detect/classifier_test.go b/scanners/subdomain-takeover/internal/detect/classifier_test.go new file mode 100644 index 0000000000..221c37ec25 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/classifier_test.go @@ -0,0 +1,132 @@ +package detect + +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/rs/zerolog" +) + +func TestClassify_ExpectedBehavior(t *testing.T) { + cnameFPs := []CNAMEProviderFingerprint{ + {Name: "Azure", Cname: []string{"azurewebsites.net"}, Nxdomain: true, Fingerprint: "unused"}, + {Name: "Ghost", Cname: []string{"ghost.io"}, Nxdomain: false, Fingerprint: "ghost 404", Mode: FingerprintModeLiteral}, + } + nsFPs := []NSProviderFingerprint{ + {Name: "RiskyDNS", Status: NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, + {Name: "SafeDNS", Status: NSStatusNotVulnerable, HostPatterns: []string{"*.safe-dns.net"}}, + } + + source := fakeSource{cname: cnameFPs, ns: nsFPs} + + t.Run("emits cname and ns findings when both are exploitable", func(t *testing.T) { + input := model.Input{ + DomainKey: "k1", + Results: model.ScanResults{ + Domain: strPtr("a.example.ca"), + CnameRecord: strPtr("a.example.ca. 300 IN CNAME foo.azurewebsites.net."), + NsDelegations: &model.NsDelegations{ + Hosts: []string{"ns1.risky-dns.net"}, + Delegation: model.Delegation{ + LameType: "partial", + }, + }, + }, + } + + findings, err := Classify(input, fakeMatcher{}, source, zerolog.Nop()) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + if len(findings) != 2 { + t.Fatalf("expected 2 findings, got %d", len(findings)) + } + + var sawCNAME, sawNS bool + for _, f := range findings { + if f.RecordType == model.RecordTypeCNAME { + sawCNAME = true + if f.ReasonCode != string(ReasonCNAMEDanglingNXDOMAIN) { + t.Fatalf("unexpected cname reason: %q", f.ReasonCode) + } + } + if f.RecordType == model.RecordTypeNS { + sawNS = true + if f.ReasonCode != string(ReasonNSPartialLameProviderVulnerable) { + t.Fatalf("unexpected ns reason: %q", f.ReasonCode) + } + } + } + if !sawCNAME || !sawNS { + t.Fatalf("missing expected finding types: cname=%v ns=%v", sawCNAME, sawNS) + } + }) + + t.Run("suppresses non-exploitable ns matches", func(t *testing.T) { + input := model.Input{ + DomainKey: "k2", + Results: model.ScanResults{ + Domain: strPtr("b.example.ca"), + CnameRecord: strPtr("b.example.ca. 300 IN CNAME foo.ghost.io."), + NsDelegations: &model.NsDelegations{ + Hosts: []string{"ns1.safe-dns.net"}, + Delegation: model.Delegation{ + LameType: "full", + }, + }, + }, + } + + matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode FingerprintMode) bool { + return true + }} + + findings, err := Classify(input, matcher, source, zerolog.Nop()) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + + if len(findings) != 1 { + t.Fatalf("expected 1 finding, got %d", len(findings)) + } + if findings[0].RecordType != model.RecordTypeCNAME { + t.Fatalf("expected cname-only finding, got %s", findings[0].RecordType) + } + }) + + t.Run("returns no findings and no panic when evidence absent", func(t *testing.T) { + input := model.Input{DomainKey: "k3", Results: model.ScanResults{}} + findings, err := Classify(input, fakeMatcher{}, source, zerolog.Nop()) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + if len(findings) != 0 { + t.Fatalf("expected no findings, got %d", len(findings)) + } + }) +} + +func TestClassifier_MethodDefaults(t *testing.T) { + classifier := NewClassifier(nil) + if classifier == nil { + t.Fatal("expected classifier") + } + + input := model.Input{DomainKey: "k", Results: model.ScanResults{}} + findings, err := classifier.Classify(input) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + if len(findings) != 0 { + t.Fatalf("expected no findings, got %d", len(findings)) + } + + custom := NewClassifierWithSource(fakeMatcher{}, fakeSource{}) + if custom == nil { + t.Fatal("expected classifier with source") + } + + if custom.WithLogger(zerolog.Nop()) != custom { + t.Fatal("WithLogger should return same classifier pointer") + } +} diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules_test.go b/scanners/subdomain-takeover/internal/detect/cname_rules_test.go new file mode 100644 index 0000000000..d3a85d9006 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/cname_rules_test.go @@ -0,0 +1,102 @@ +package detect + +import "testing" + +func TestMatchCNAMEFingerprints(t *testing.T) { + fps := []CNAMEProviderFingerprint{ + { + Name: "Azure", + Cname: []string{"azurewebsites.net"}, + Nxdomain: true, + Fingerprint: "This web app is stopped", + }, + { + Name: "Ghost", + Cname: []string{"ghost.io"}, + Nxdomain: false, + Fingerprint: "The thing you were looking for is no longer here", + Mode: FingerprintModeLiteral, + }, + } + + t.Run("returns nil when no target match", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "foo.not-a-provider.net", NoResolve: true} + if got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{}); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("nxdomain provider emits dangling reason when unresolved", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "foo.azurewebsites.net", NoResolve: true} + got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{}) + if got == nil { + t.Fatal("expected hit, got nil") + } + if !got.Matched { + t.Fatal("expected matched=true") + } + if got.ReasonCode != ReasonCNAMEDanglingNXDOMAIN { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("nxdomain provider suppressed when resolve evidence exists", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "foo.azurewebsites.net", NoResolve: false} + got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{}) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Matched { + t.Fatal("expected matched=false") + } + if got.ReasonCode != ReasonCNAMETargetMatchMissingNXDOMAIN { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("body fingerprint provider emits when matcher returns true", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "blog.ghost.io", NoResolve: false} + matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode FingerprintMode) bool { + if domain != "a.example.ca" || mode != FingerprintModeLiteral { + t.Fatalf("unexpected matcher args: domain=%q mode=%q", domain, mode) + } + return true + }} + got := MatchCNAMEFingerprints(evidence, fps, matcher) + if got == nil { + t.Fatal("expected hit, got nil") + } + if !got.Matched { + t.Fatal("expected matched=true") + } + if got.ReasonCode != ReasonCNAMEProviderFingerprintBodyMatch { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("body fingerprint provider suppressed when matcher false", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "blog.ghost.io", NoResolve: false} + got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{containsFn: func(string, string, FingerprintMode) bool { return false }}) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Matched { + t.Fatal("expected matched=false") + } + if got.ReasonCode != ReasonCNAMETargetMatchMissingBodyFP { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) +} + +func TestShouldEmitCNAME(t *testing.T) { + if ShouldEmitCNAME(nil) { + t.Fatal("expected false for nil hit") + } + if ShouldEmitCNAME(&CNAMEHit{Matched: false}) { + t.Fatal("expected false for unmatched hit") + } + if !ShouldEmitCNAME(&CNAMEHit{Matched: true}) { + t.Fatal("expected true for matched hit") + } +} diff --git a/scanners/subdomain-takeover/internal/detect/confidence_test.go b/scanners/subdomain-takeover/internal/detect/confidence_test.go new file mode 100644 index 0000000000..d7efdd2061 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/confidence_test.go @@ -0,0 +1,26 @@ +package detect + +import "testing" + +func TestConfidenceForReason(t *testing.T) { + tests := []struct { + reason ReasonCode + want string + }{ + {ReasonCNAMEDanglingNXDOMAIN, ConfidenceProbable}, + {ReasonCNAMEProviderFingerprintBodyMatch, ConfidenceProbable}, + {ReasonCNAMETargetMatchMissingNXDOMAIN, ConfidenceSuspected}, + {ReasonCNAMETargetMatchMissingBodyFP, ConfidenceSuspected}, + {ReasonNSFullLameProviderVulnerable, ConfidenceConfirmed}, + {ReasonNSPartialLameProviderVulnerable, ConfidenceProbable}, + {ReasonNSLameProviderUnknown, ConfidenceSuspected}, + {ReasonNSProviderMatchOnly, ConfidenceSuspected}, + {ReasonCode("UNKNOWN_REASON"), ConfidenceSuspected}, + } + + for _, tt := range tests { + if got := ConfidenceForReason(tt.reason); got != tt.want { + t.Fatalf("ConfidenceForReason(%q)=%q want=%q", tt.reason, got, tt.want) + } + } +} diff --git a/scanners/subdomain-takeover/internal/detect/evidence_test.go b/scanners/subdomain-takeover/internal/detect/evidence_test.go new file mode 100644 index 0000000000..1dc1006fe9 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/evidence_test.go @@ -0,0 +1,128 @@ +package detect + +import ( + "reflect" + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +func TestExtractCNAMEEvidence(t *testing.T) { + t.Run("returns nil when cname record missing", func(t *testing.T) { + results := minimalResults("app.example.ca") + if got := ExtractCNAMEEvidence(results); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil when domain missing", func(t *testing.T) { + record := "a.example.ca. 300 IN CNAME foo.azurewebsites.net." + results := model.ScanResults{CnameRecord: &record} + if got := ExtractCNAMEEvidence(results); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("extracts normalized target and noresolve true", func(t *testing.T) { + record := "A.Example.CA. 300 IN CNAME Foo.AzureWebsites.NET." + results := minimalResults("app.example.ca") + results.CnameRecord = &record + + got := ExtractCNAMEEvidence(results) + if got == nil { + t.Fatal("expected evidence, got nil") + } + + if got.Domain != "app.example.ca" { + t.Fatalf("unexpected domain: %q", got.Domain) + } + if got.Target != "foo.azurewebsites.net" { + t.Fatalf("unexpected target: %q", got.Target) + } + if !got.NoResolve { + t.Fatal("expected NoResolve=true") + } + }) + + t.Run("sets noresolve false when resolve chain present", func(t *testing.T) { + record := "a.example.ca. 300 IN CNAME foo.azurewebsites.net." + results := minimalResults("app.example.ca") + results.CnameRecord = &record + results.ResolveChain = [][]string{{"a.example.ca CNAME foo.azurewebsites.net"}} + + got := ExtractCNAMEEvidence(results) + if got == nil { + t.Fatal("expected evidence, got nil") + } + if got.NoResolve { + t.Fatal("expected NoResolve=false") + } + }) +} + +func TestExtractNSEvidence(t *testing.T) { + t.Run("returns nil when domain missing", func(t *testing.T) { + input := model.ScanResults{NsDelegations: &model.NsDelegations{Hosts: []string{"ns1.example.net"}}} + if got := ExtractNSEvidence(input); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil when ns delegations missing", func(t *testing.T) { + input := minimalResults("app.example.ca") + if got := ExtractNSEvidence(input); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil when no hosts", func(t *testing.T) { + input := minimalResults("app.example.ca") + input.NsDelegations = &model.NsDelegations{} + if got := ExtractNSEvidence(input); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("normalizes, sorts, and deduplicates hosts", func(t *testing.T) { + input := minimalResults("app.example.ca") + input.NsDelegations = &model.NsDelegations{ + Hosts: []string{"NS2.Example.Net.", "ns1.example.net", "ns2.example.net", ".ns1.example.net."}, + Delegation: model.Delegation{ + LameType: "partial", + }, + } + + got := ExtractNSEvidence(input) + if got == nil { + t.Fatal("expected evidence, got nil") + } + + expectedHosts := []string{"ns1.example.net", "ns2.example.net"} + if !reflect.DeepEqual(got.NSHosts, expectedHosts) { + t.Fatalf("unexpected hosts: got=%v want=%v", got.NSHosts, expectedHosts) + } + if got.Domain != "app.example.ca" { + t.Fatalf("unexpected domain: %q", got.Domain) + } + if got.NSDelegations.Delegation.LameType != "partial" { + t.Fatalf("unexpected lame type: %q", got.NSDelegations.Delegation.LameType) + } + }) +} + +func TestNormalizeHostAndParseHelpers(t *testing.T) { + if got := normalizeHost(".NS1.Example.NET."); got != "ns1.example.net" { + t.Fatalf("normalizeHost mismatch: %q", got) + } + + record := "x.example.ca. 300 IN CNAME target.provider.net." + if got := parseCname(record); got != "target.provider.net" { + t.Fatalf("parseCname mismatch: %q", got) + } + + hosts := []string{"ns3.example.net", "NS1.example.net.", "ns1.example.net", "ns2.example.net"} + expected := []string{"ns1.example.net", "ns2.example.net", "ns3.example.net"} + if got := parseHostnames(hosts); !reflect.DeepEqual(got, expected) { + t.Fatalf("parseHostnames mismatch: got=%v want=%v", got, expected) + } +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_loader_test.go b/scanners/subdomain-takeover/internal/detect/fingerprint_loader_test.go new file mode 100644 index 0000000000..726d95b4d9 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_loader_test.go @@ -0,0 +1,47 @@ +package detect + +import ( + "sync" + "testing" + + "github.com/rs/zerolog" +) + +func TestLoadFingerprints(t *testing.T) { + origOnce := loadFingerprintsOnce + origErr := loadFingerprintsErr + origCNAME := CNAMEProviderFingerprints + origNS := NSProviderFingerprints + + loadFingerprintsOnce = sync.Once{} + loadFingerprintsErr = nil + CNAMEProviderFingerprints = nil + NSProviderFingerprints = nil + + t.Cleanup(func() { + loadFingerprintsOnce = origOnce + loadFingerprintsErr = origErr + CNAMEProviderFingerprints = origCNAME + NSProviderFingerprints = origNS + }) + + err := LoadFingerprints(zerolog.Nop()) + if err != nil { + t.Fatalf("LoadFingerprints returned error: %v", err) + } + if len(CNAMEProviderFingerprints) == 0 { + t.Fatal("expected cname fingerprints to be loaded") + } + if len(NSProviderFingerprints) == 0 { + t.Fatal("expected ns fingerprints to be loaded") + } +} + +func TestValidateFingerprintHelpers(t *testing.T) { + if err := validateCNAMEFingerprints([]CNAMEProviderFingerprint{{Name: "", Cname: []string{"x"}, Fingerprint: "fp"}}); err == nil { + t.Fatal("expected error for missing cname fingerprint name") + } + if err := validateNSFingerprints([]NSProviderFingerprint{{Name: "ok", HostPatterns: []string{""}}}); err == nil { + t.Fatal("expected error for empty ns host pattern") + } +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go b/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go new file mode 100644 index 0000000000..fa9c8160ec --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go @@ -0,0 +1,23 @@ +package detect + +import "testing" + +func TestGlobalFingerprintSource(t *testing.T) { + origCNAME := CNAMEProviderFingerprints + origNS := NSProviderFingerprints + t.Cleanup(func() { + CNAMEProviderFingerprints = origCNAME + NSProviderFingerprints = origNS + }) + + CNAMEProviderFingerprints = []CNAMEProviderFingerprint{{Name: "A"}} + NSProviderFingerprints = []NSProviderFingerprint{{Name: "B"}} + + src := GlobalFingerprintSource{} + if got := src.CNAME(); len(got) != 1 || got[0].Name != "A" { + t.Fatalf("unexpected cname source: %+v", got) + } + if got := src.NS(); len(got) != 1 || got[0].Name != "B" { + t.Fatalf("unexpected ns source: %+v", got) + } +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints_test.go b/scanners/subdomain-takeover/internal/detect/fingerprints_test.go new file mode 100644 index 0000000000..a31934890e --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprints_test.go @@ -0,0 +1,47 @@ +package detect + +import "testing" + +func TestContainsNSHostAndWildcardMatch(t *testing.T) { + fp := NSProviderFingerprint{Name: "Test", HostPatterns: []string{"*.example.net", "ns.fixed.io"}} + + if !fp.ContainsNSHost("NS1.EXAMPLE.NET.") { + t.Fatal("expected wildcard match for ns1.example.net") + } + if !fp.ContainsNSHost("ns.fixed.io") { + t.Fatal("expected exact match for ns.fixed.io") + } + if fp.ContainsNSHost("other.net") { + t.Fatal("did not expect match for other.net") + } + + if !wildcardHostMatch("abc*xyz*io", "abc-123-xyz-final-io") { + t.Fatal("expected multi-star wildcard to match") + } + if wildcardHostMatch("abc*xyz", "zabc123xyz") { + t.Fatal("did not expect prefix mismatch to match") + } +} + +func TestContainsTargetAndNormalizeMode(t *testing.T) { + fp := CNAMEProviderFingerprint{Cname: []string{"azurewebsites.net"}} + if !fp.ContainsTarget("foo.azurewebsites.net") { + t.Fatal("expected suffix match") + } + if fp.ContainsTarget("foo.example.net") { + t.Fatal("did not expect suffix mismatch") + } + + if got := normalizeFingerprintMode(FingerprintModeLiteral, "foo"); got != FingerprintModeLiteral { + t.Fatalf("unexpected mode: %q", got) + } + if got := normalizeFingerprintMode("", "service unavailable"); got != FingerprintModeLiteral { + t.Fatalf("unexpected inferred mode: %q", got) + } + if got := normalizeFingerprintMode("", "Error: .* not found"); got != FingerprintModeRegex { + t.Fatalf("unexpected inferred mode: %q", got) + } + if got := normalizeFingerprintMode("", "foo\\d+"); got != FingerprintModeRegex { + t.Fatalf("unexpected inferred mode: %q", got) + } +} diff --git a/scanners/subdomain-takeover/internal/detect/logger_test.go b/scanners/subdomain-takeover/internal/detect/logger_test.go new file mode 100644 index 0000000000..2487426271 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/logger_test.go @@ -0,0 +1,15 @@ +package detect + +import ( + "io" + "testing" + + "github.com/rs/zerolog" +) + +func TestSetLogger(t *testing.T) { + SetLogger(zerolog.New(io.Discard)) + if got := detectLogger.GetLevel(); got == zerolog.Disabled { + t.Fatalf("expected configured logger, got level=%s", got) + } +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go new file mode 100644 index 0000000000..3521974672 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go @@ -0,0 +1,62 @@ +package detect + +import "testing" + +func TestGetNSHijackReasonCode(t *testing.T) { + tests := []struct { + name string + lameType string + status NSProviderStatus + want ReasonCode + }{ + {name: "full vulnerable", lameType: "full", status: NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, + {name: "partial vulnerable", lameType: "partial", status: NSStatusVulnerable, want: ReasonNSPartialLameProviderVulnerable}, + {name: "full vulnerable with purchase", lameType: "full", status: NSStatusVulnerableWithPurchase, want: ReasonNSFullLameProviderVulnerable}, + {name: "full not vulnerable", lameType: "full", status: NSStatusNotVulnerable, want: ReasonNSLameProviderUnknown}, + {name: "partial registration closed", lameType: "partial", status: NSStatusRegistrationClosed, want: ReasonNSLameProviderUnknown}, + {name: "unknown lame type", lameType: "none", status: NSStatusVulnerable, want: ReasonNSProviderMatchOnly}, + {name: "normalized lame type", lameType: " FULL ", status: NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getNSHijackReasonCode(tt.lameType, tt.status) + if got != tt.want { + t.Fatalf("getNSHijackReasonCode()=%q want=%q", got, tt.want) + } + }) + } +} + +func TestNSReasoningHelpers(t *testing.T) { + if !isExploitableProviderStatus(NSStatusVulnerable) { + t.Fatal("expected vulnerable to be exploitable") + } + if !isExploitableProviderStatus(NSStatusVulnerableWithPurchase) { + t.Fatal("expected vulnerable_with_purchase to be exploitable") + } + if isExploitableProviderStatus(NSStatusNotVulnerable) { + t.Fatal("expected not_vulnerable to be non-exploitable") + } + + if got := normalizeLameType(" Partial "); got != "partial" { + t.Fatalf("normalizeLameType mismatch: %q", got) + } + + if !isNSMatch(ReasonNSFullLameProviderVulnerable) || !isNSMatch(ReasonNSPartialLameProviderVulnerable) { + t.Fatal("expected vulnerable reason codes to be emittable") + } + if isNSMatch(ReasonNSLameProviderUnknown) { + t.Fatal("expected unknown reason not to be emittable") + } + + if nsReasonRank(ReasonNSFullLameProviderVulnerable) <= nsReasonRank(ReasonNSPartialLameProviderVulnerable) { + t.Fatal("expected full lame rank > partial lame rank") + } + if nsReasonRank(ReasonNSPartialLameProviderVulnerable) <= nsReasonRank(ReasonNSLameProviderUnknown) { + t.Fatal("expected partial lame rank > unknown rank") + } + if nsReasonRank(ReasonNSLameProviderUnknown) <= nsReasonRank(ReasonNSProviderMatchOnly) { + t.Fatal("expected unknown rank > provider-only rank") + } +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go new file mode 100644 index 0000000000..c12671323d --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go @@ -0,0 +1,110 @@ +package detect + +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +func TestMatchNSProviderRules(t *testing.T) { + nsFP := []NSProviderFingerprint{ + {Name: "UnknownDNS", Status: NSStatusNotVulnerable, HostPatterns: []string{"*.unknown-dns.net"}}, + {Name: "RiskyDNS", Status: NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, + } + + t.Run("returns nil for missing hosts", func(t *testing.T) { + evidence := NSEvidence{Domain: "a.example.ca"} + if got := MatchNSProviderRules(evidence, nsFP); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil for missing fingerprints", func(t *testing.T) { + evidence := NSEvidence{Domain: "a.example.ca", NSHosts: []string{"ns1.risky-dns.net"}} + if got := MatchNSProviderRules(evidence, nil); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil when no provider matches", func(t *testing.T) { + evidence := NSEvidence{Domain: "a.example.ca", NSHosts: []string{"ns1.nomatch.net"}} + if got := MatchNSProviderRules(evidence, nsFP); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("matches vulnerable full lame and emits correct reason", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.risky-dns.net"}, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + got := MatchNSProviderRules(evidence, nsFP) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Provider != "RiskyDNS" { + t.Fatalf("unexpected provider: %q", got.Provider) + } + if got.ReasonCode != ReasonNSFullLameProviderVulnerable { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if !got.Matched { + t.Fatal("expected matched=true") + } + }) + + t.Run("ranks vulnerable higher than unknown", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.unknown-dns.net", "ns1.risky-dns.net"}, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "partial", + }}, + } + got := MatchNSProviderRules(evidence, nsFP) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Provider != "RiskyDNS" { + t.Fatalf("expected risky provider to win ranking, got %q", got.Provider) + } + if got.ReasonCode != ReasonNSPartialLameProviderVulnerable { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("non-vulnerable full lame is classified but not emittable", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.unknown-dns.net"}, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + got := MatchNSProviderRules(evidence, nsFP) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.ReasonCode != ReasonNSLameProviderUnknown { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if got.Matched { + t.Fatal("expected matched=false for non-vulnerable state") + } + }) +} + +func TestShouldEmitNSHijack(t *testing.T) { + if ShouldEmitNSHijack(nil) { + t.Fatal("expected false for nil hit") + } + if ShouldEmitNSHijack(&NSHit{Matched: false}) { + t.Fatal("expected false for unmatched hit") + } + if !ShouldEmitNSHijack(&NSHit{Matched: true}) { + t.Fatal("expected true for matched hit") + } +} diff --git a/scanners/subdomain-takeover/internal/detect/test_helpers_test.go b/scanners/subdomain-takeover/internal/detect/test_helpers_test.go new file mode 100644 index 0000000000..f73fcca80f --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/test_helpers_test.go @@ -0,0 +1,32 @@ +package detect + +import ( + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +type fakeMatcher struct { + containsFn func(domain string, fingerprint string, mode FingerprintMode) bool +} + +func (f fakeMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { + if f.containsFn == nil { + return false + } + return f.containsFn(domain, fingerprint, mode) +} + +type fakeSource struct { + cname []CNAMEProviderFingerprint + ns []NSProviderFingerprint +} + +func (f fakeSource) CNAME() []CNAMEProviderFingerprint { return f.cname } +func (f fakeSource) NS() []NSProviderFingerprint { return f.ns } + +func strPtr(v string) *string { return &v } + +func boolPtr(v bool) *bool { return &v } + +func minimalResults(domain string) model.ScanResults { + return model.ScanResults{Domain: strPtr(domain)} +} From 2a63ac44a4c519f563f816881012cb46fd019498 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 17:44:03 -0300 Subject: [PATCH 36/65] app logic tests --- .../internal/app/runner_test.go | 87 ++++++++ .../internal/app/worker_test.go | 205 ++++++++++++++++++ 2 files changed, 292 insertions(+) create mode 100644 scanners/subdomain-takeover/internal/app/runner_test.go create mode 100644 scanners/subdomain-takeover/internal/app/worker_test.go diff --git a/scanners/subdomain-takeover/internal/app/runner_test.go b/scanners/subdomain-takeover/internal/app/runner_test.go new file mode 100644 index 0000000000..d66c0e0b16 --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/runner_test.go @@ -0,0 +1,87 @@ +package app + +import ( + "context" + "sync" + "testing" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type fakeMessagesIter struct { + mu sync.Mutex + msgs []jetstream.Msg + err error + idx int + stopCalled bool +} + +func (f *fakeMessagesIter) Next(...jetstream.NextOpt) (jetstream.Msg, error) { + f.mu.Lock() + defer f.mu.Unlock() + + if f.err != nil { + return nil, f.err + } + if f.idx >= len(f.msgs) { + return nil, jetstream.ErrMsgIteratorClosed + } + m := f.msgs[f.idx] + f.idx++ + return m, nil +} + +func (f *fakeMessagesIter) Stop() { f.stopCalled = true } +func (f *fakeMessagesIter) Drain() {} + +type fakeHandler struct { + mu sync.Mutex + count int +} + +func (f *fakeHandler) Handle(context.Context, jetstream.Msg) error { + f.mu.Lock() + f.count++ + f.mu.Unlock() + return nil +} + +// nolint:revive // nats Conn fields are unexported; this fake nil connection path only. +func TestRun_ReturnsImmediatelyWhenConnectionUnhealthy(t *testing.T) { + iter := &fakeMessagesIter{} + h := &fakeHandler{} + + deps := RunnerDeps{ + Logger: zerolog.Nop(), + WorkerCount: 2, + Iter: iter, + Worker: h, + NC: nil, + } + + Run(context.Background(), deps) + + if h.count != 0 { + t.Fatalf("expected no handled messages, got %d", h.count) + } +} + +func TestRun_ClampsWorkerCountBelowOne(t *testing.T) { + iter := &fakeMessagesIter{} + h := &fakeHandler{} + + deps := RunnerDeps{ + Logger: zerolog.Nop(), + WorkerCount: 0, + Iter: iter, + Worker: h, + NC: (*nats.Conn)(nil), + } + + Run(context.Background(), deps) + if h.count != 0 { + t.Fatalf("expected no handled messages, got %d", h.count) + } +} diff --git a/scanners/subdomain-takeover/internal/app/worker_test.go b/scanners/subdomain-takeover/internal/app/worker_test.go new file mode 100644 index 0000000000..c36915185f --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/worker_test.go @@ -0,0 +1,205 @@ +package app + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type fakePublisher struct { + err error + published []model.Finding +} + +func (f *fakePublisher) Publish(_ context.Context, finding model.Finding) error { + if f.err != nil { + return f.err + } + f.published = append(f.published, finding) + return nil +} + +type fakeClassifier struct { + findings []model.Finding + err error +} + +func (f fakeClassifier) Classify(model.Input) ([]model.Finding, error) { + if f.err != nil { + return nil, f.err + } + return f.findings, nil +} + +type fakeJSMsg struct { + data []byte + subject string + + ackErr error + nakErr error + termErr error + + ackCount int + nakCount int + termCount int +} + +func (m *fakeJSMsg) Metadata() (*jetstream.MsgMetadata, error) { return nil, nil } +func (m *fakeJSMsg) Data() []byte { return m.data } +func (m *fakeJSMsg) Headers() nats.Header { return nil } +func (m *fakeJSMsg) Subject() string { return m.subject } +func (m *fakeJSMsg) Reply() string { return "" } + +func (m *fakeJSMsg) Ack() error { + m.ackCount++ + return m.ackErr +} + +func (m *fakeJSMsg) DoubleAck(context.Context) error { return nil } + +func (m *fakeJSMsg) Nak() error { + m.nakCount++ + return m.nakErr +} + +func (m *fakeJSMsg) NakWithDelay(time.Duration) error { return nil } +func (m *fakeJSMsg) InProgress() error { return nil } + +func (m *fakeJSMsg) Term() error { + m.termCount++ + return m.termErr +} + +func (m *fakeJSMsg) TermWithReason(string) error { return nil } + +func TestWorkerHandle(t *testing.T) { + logger := zerolog.Nop() + + t.Run("decode error terminates message", func(t *testing.T) { + pub := &fakePublisher{} + classifier := fakeClassifier{} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{data: []byte("{not-json"), subject: "scans.dns_scanner_results"} + err := worker.Handle(context.Background(), msg) + if err == nil { + t.Fatal("expected decode error") + } + if msg.termCount != 1 { + t.Fatalf("expected term once, got %d", msg.termCount) + } + if msg.nakCount != 0 || msg.ackCount != 0 { + t.Fatalf("unexpected ack/nak counts: ack=%d nak=%d", msg.ackCount, msg.nakCount) + } + }) + + t.Run("classification error naks message", func(t *testing.T) { + pub := &fakePublisher{} + classifier := fakeClassifier{err: errors.New("classify failed")} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"} + err := worker.Handle(context.Background(), msg) + if err == nil { + t.Fatal("expected classification error") + } + if msg.nakCount != 1 { + t.Fatalf("expected nak once, got %d", msg.nakCount) + } + if msg.ackCount != 0 || msg.termCount != 0 { + t.Fatalf("unexpected ack/term counts: ack=%d term=%d", msg.ackCount, msg.termCount) + } + }) + + t.Run("publish error naks message", func(t *testing.T) { + pub := &fakePublisher{err: errors.New("publish failed")} + classifier := fakeClassifier{findings: []model.Finding{{Domain: "a.example.ca"}}} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"} + err := worker.Handle(context.Background(), msg) + if err == nil { + t.Fatal("expected publish error") + } + if msg.nakCount != 1 { + t.Fatalf("expected nak once, got %d", msg.nakCount) + } + if msg.ackCount != 0 || msg.termCount != 0 { + t.Fatalf("unexpected ack/term counts: ack=%d term=%d", msg.ackCount, msg.termCount) + } + }) + + t.Run("successful processing publishes all findings and acks", func(t *testing.T) { + pub := &fakePublisher{} + classifier := fakeClassifier{findings: []model.Finding{{Domain: "a.example.ca"}, {Domain: "b.example.ca"}}} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"} + err := worker.Handle(context.Background(), msg) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(pub.published) != 2 { + t.Fatalf("expected 2 published findings, got %d", len(pub.published)) + } + if msg.ackCount != 1 { + t.Fatalf("expected ack once, got %d", msg.ackCount) + } + if msg.nakCount != 0 || msg.termCount != 0 { + t.Fatalf("unexpected nak/term counts: nak=%d term=%d", msg.nakCount, msg.termCount) + } + }) + + t.Run("ack failure returns error", func(t *testing.T) { + pub := &fakePublisher{} + classifier := fakeClassifier{} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{ + data: []byte(`{"domain_key":"k","results":{}}`), + subject: "scans.dns_scanner_results", + ackErr: errors.New("ack failed"), + } + + err := worker.Handle(context.Background(), msg) + if err == nil { + t.Fatal("expected ack error") + } + if msg.ackCount != 1 { + t.Fatalf("expected ack once, got %d", msg.ackCount) + } + }) + + t.Run("decode error still returned when term fails", func(t *testing.T) { + pub := &fakePublisher{} + classifier := fakeClassifier{} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{ + data: []byte("{bad-json"), + subject: "scans.dns_scanner_results", + termErr: errors.New("term failed"), + } + + err := worker.Handle(context.Background(), msg) + if err == nil { + t.Fatal("expected decode error") + } + if msg.termCount != 1 { + t.Fatalf("expected term once, got %d", msg.termCount) + } + }) +} + +func TestDecodeScan_TrimsTrailingNewline(t *testing.T) { + _, err := decodeScan([]byte("{\"domain_key\":\"k\",\"results\":{}}\n")) + if err != nil { + t.Fatalf("expected decode success, got error: %v", err) + } +} From 6be58701408d03bbaade6ee9153f15cbca1f0938 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 22 Jul 2026 17:44:16 -0300 Subject: [PATCH 37/65] messaging logic tests --- .../internal/messaging/health_test.go | 13 +++ .../internal/messaging/publisher.go | 8 +- .../internal/messaging/publisher_test.go | 93 +++++++++++++++++++ 3 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/messaging/health_test.go create mode 100644 scanners/subdomain-takeover/internal/messaging/publisher_test.go diff --git a/scanners/subdomain-takeover/internal/messaging/health_test.go b/scanners/subdomain-takeover/internal/messaging/health_test.go new file mode 100644 index 0000000000..de46741d53 --- /dev/null +++ b/scanners/subdomain-takeover/internal/messaging/health_test.go @@ -0,0 +1,13 @@ +package messaging + +import "testing" + +func TestCheckConnection(t *testing.T) { + err := CheckConnection(nil) + if err == nil { + t.Fatal("expected error for nil connection") + } + if err.Error() != "nats connection is nil" { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go index 5626f45983..7ba6e866be 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -11,11 +11,15 @@ import ( type Publisher struct { logger zerolog.Logger - js jetstream.JetStream + js publishClient subject string } -func NewPublisher(logger zerolog.Logger, js jetstream.JetStream, subject string) *Publisher { +type publishClient interface { + Publish(ctx context.Context, subj string, data []byte, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error) +} + +func NewPublisher(logger zerolog.Logger, js publishClient, subject string) *Publisher { return &Publisher{logger: logger, js: js, subject: subject} } diff --git a/scanners/subdomain-takeover/internal/messaging/publisher_test.go b/scanners/subdomain-takeover/internal/messaging/publisher_test.go new file mode 100644 index 0000000000..38de994a00 --- /dev/null +++ b/scanners/subdomain-takeover/internal/messaging/publisher_test.go @@ -0,0 +1,93 @@ +package messaging + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type fakePublishClient struct { + err error + ack *jetstream.PubAck + called int + lastSubj string + lastBytes []byte +} + +func (f *fakePublishClient) Publish(_ context.Context, subj string, data []byte, _ ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + f.called++ + f.lastSubj = subj + f.lastBytes = append([]byte(nil), data...) + if f.err != nil { + return nil, f.err + } + if f.ack != nil { + return f.ack, nil + } + return &jetstream.PubAck{}, nil +} + +func TestNewPublisher(t *testing.T) { + client := &fakePublishClient{} + p := NewPublisher(zerolog.Nop(), client, "scans.findings.upsert") + if p == nil { + t.Fatal("expected publisher instance") + } + if p.subject != "scans.findings.upsert" { + t.Fatalf("unexpected subject: %q", p.subject) + } +} + +func TestPublisherPublish(t *testing.T) { + finding := model.Finding{ + Domain: "a.example.ca", + DomainKey: "123", + RecordType: model.RecordTypeNS, + Target: "ns1.risky-dns.net", + Provider: "RiskyDNS", + Confidence: "probable", + ReasonCode: "NS_PARTIAL_LAME_PROVIDER_VULNERABLE", + } + + t.Run("publishes serialized finding", func(t *testing.T) { + client := &fakePublishClient{} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.upsert"} + + err := p.Publish(context.Background(), finding) + if err != nil { + t.Fatalf("unexpected publish error: %v", err) + } + if client.called != 1 { + t.Fatalf("expected one publish call, got %d", client.called) + } + if client.lastSubj != "scans.findings.upsert" { + t.Fatalf("unexpected subject: %q", client.lastSubj) + } + + var got model.Finding + if err := json.Unmarshal(client.lastBytes, &got); err != nil { + t.Fatalf("payload not valid json: %v", err) + } + if got != finding { + t.Fatalf("unexpected payload: got=%+v want=%+v", got, finding) + } + }) + + t.Run("returns publish client error", func(t *testing.T) { + client := &fakePublishClient{err: errors.New("publish failed")} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.upsert"} + + err := p.Publish(context.Background(), finding) + if err == nil { + t.Fatal("expected publish error") + } + if client.called != 1 { + t.Fatalf("expected one publish call, got %d", client.called) + } + }) +} From 65ee95ddcd36eb6d229a0f2214c10823632c2264 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Thu, 23 Jul 2026 12:26:12 -0300 Subject: [PATCH 38/65] add CNAME rtype to record_exists check --- scanners/dns-scanner/dns_scanner/dns_scanner.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index 40567ba136..3a2c6fb5a3 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -212,7 +212,12 @@ def scan_domain(domain, dkim_selectors=None): # Check if domain exists dns_answer_return_types = [] - for query_type in [dns.rdatatype.A, dns.rdatatype.SOA, dns.rdatatype.NS]: + for query_type in [ + dns.rdatatype.A, + dns.rdatatype.CNAME, + dns.rdatatype.SOA, + dns.rdatatype.NS, + ]: rtype = get_dns_return_type(domain, query_type) if rtype == "NOERROR": dns_answer_return_types.append(rtype) @@ -329,7 +334,7 @@ def scan_domain(domain, dkim_selectors=None): registrar_domain = scan_result.base_domain or zone_apex or domain scan_result.registrar_context = get_registrar_context( base_domain=registrar_domain, - ns_hosts=ns_delegations.get("ns_hosts", []), + ns_hosts=scan_result.ns_delegations.get("ns_hosts", []), ) # If no MX records are found (with warnings), but there are CNAME records, check the CNAME target for MX records From 64f8fd950b1ce8f2dd171e1b4581789a74515726 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Thu, 23 Jul 2026 17:12:56 -0300 Subject: [PATCH 39/65] refactor fingerprints into separate package --- scanners/subdomain-takeover/README.md | 19 +++++++-- .../subdomain-takeover/cmd/service/main.go | 3 +- .../internal/app/runner_test.go | 2 +- .../internal/app/worker_test.go | 2 +- .../internal/detect/body_matcher.go | 12 +++--- .../internal/detect/body_matcher_test.go | 16 ++++---- .../internal/detect/classifier.go | 8 ++-- .../internal/detect/classifier_test.go | 13 +++--- .../internal/detect/cname_rules.go | 8 ++-- .../internal/detect/cname_rules_test.go | 16 +++++--- .../internal/detect/fingerprint_source.go | 14 ++++--- .../detect/fingerprint_source_test.go | 27 ++++++------ .../internal/detect/ns_reasoning.go | 12 ++++-- .../internal/detect/ns_reasoning_test.go | 28 +++++++------ .../internal/detect/ns_rules.go | 18 ++++---- .../internal/detect/ns_rules_test.go | 7 ++-- .../internal/detect/test_helpers_test.go | 13 +++--- .../data/cname_fingerprints.json | 0 .../data/ns_fingerprints.json | 0 .../loader.go} | 41 ++++++++++++------- .../loader_test.go} | 35 ++++++++-------- .../fingerprints.go => fingerprints/types.go} | 7 +--- .../types_test.go} | 10 ++--- 23 files changed, 178 insertions(+), 133 deletions(-) rename scanners/subdomain-takeover/internal/{detect => fingerprints}/data/cname_fingerprints.json (100%) rename scanners/subdomain-takeover/internal/{detect => fingerprints}/data/ns_fingerprints.json (100%) rename scanners/subdomain-takeover/internal/{detect/fingerprint_loader.go => fingerprints/loader.go} (79%) rename scanners/subdomain-takeover/internal/{detect/fingerprint_loader_test.go => fingerprints/loader_test.go} (50%) rename scanners/subdomain-takeover/internal/{detect/fingerprints.go => fingerprints/types.go} (92%) rename scanners/subdomain-takeover/internal/{detect/fingerprints_test.go => fingerprints/types_test.go} (75%) diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md index 7e28388320..0345e1ec2c 100644 --- a/scanners/subdomain-takeover/README.md +++ b/scanners/subdomain-takeover/README.md @@ -75,10 +75,10 @@ Detection logic was split into focused files to reduce coupling: Provider fingerprint data is stored in: -- `internal/detect/data/cname_fingerprints.json` -- `internal/detect/data/ns_fingerprints.json` +- `internal/fingerprints/data/cname_fingerprints.json` +- `internal/fingerprints/data/ns_fingerprints.json` -The service embeds and validates these files at startup (`detect.LoadFingerprints`). +The service embeds and validates these files at startup (`fingerprints.Load`). For CNAME body matching: @@ -99,6 +99,19 @@ go mod tidy go run ./cmd/service ``` +## Local commands + +This service includes a local `Makefile` for common workflows: + +```bash +make help # list targets +make run # run the service +make test # run tests +make lint # fmt-check + vet +make build # build bin/subdomain-takeover +make ci # lint + test + build +``` + Environment variables: - `NATS_URL` (default: `nats://localhost:4222`) - `NATS_STREAM` (default: `SCANS`) diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 78a2500bff..816ab0cc80 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -11,6 +11,7 @@ import ( "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/bootstrap" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" "github.com/rs/zerolog" ) @@ -34,7 +35,7 @@ func main() { Str("log_level", cfg.LogLevel.String()). Msg("service configuration loaded") - if err := detect.LoadFingerprints(logger); err != nil { + if err := fingerprints.Load(logger); err != nil { logger.Fatal().Err(err).Msg("failed to load fingerprints") } diff --git a/scanners/subdomain-takeover/internal/app/runner_test.go b/scanners/subdomain-takeover/internal/app/runner_test.go index d66c0e0b16..7d076aa67d 100644 --- a/scanners/subdomain-takeover/internal/app/runner_test.go +++ b/scanners/subdomain-takeover/internal/app/runner_test.go @@ -33,7 +33,7 @@ func (f *fakeMessagesIter) Next(...jetstream.NextOpt) (jetstream.Msg, error) { return m, nil } -func (f *fakeMessagesIter) Stop() { f.stopCalled = true } +func (f *fakeMessagesIter) Stop() { f.stopCalled = true } func (f *fakeMessagesIter) Drain() {} type fakeHandler struct { diff --git a/scanners/subdomain-takeover/internal/app/worker_test.go b/scanners/subdomain-takeover/internal/app/worker_test.go index c36915185f..fc771e4e15 100644 --- a/scanners/subdomain-takeover/internal/app/worker_test.go +++ b/scanners/subdomain-takeover/internal/app/worker_test.go @@ -13,7 +13,7 @@ import ( ) type fakePublisher struct { - err error + err error published []model.Finding } diff --git a/scanners/subdomain-takeover/internal/detect/body_matcher.go b/scanners/subdomain-takeover/internal/detect/body_matcher.go index 622b2f7c51..b28d4b496e 100644 --- a/scanners/subdomain-takeover/internal/detect/body_matcher.go +++ b/scanners/subdomain-takeover/internal/detect/body_matcher.go @@ -7,10 +7,12 @@ import ( "regexp" "strings" "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" ) type BodyFingerprintMatcher interface { - Contains(domain string, fingerprint string, mode FingerprintMode) bool + Contains(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool } type HTTPBodyFingerprintMatcher struct { @@ -29,11 +31,11 @@ func NewNoopBodyFingerprintMatcher() *NoopBodyFingerprintMatcher { return &NoopBodyFingerprintMatcher{} } -func (m *NoopBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { +func (m *NoopBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { return false } -func (m *HTTPBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { +func (m *HTTPBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { if m == nil || m.client == nil { return false } @@ -51,9 +53,9 @@ func (m *HTTPBodyFingerprintMatcher) Contains(domain string, fingerprint string, } bodyText := string(body) - resolvedMode := normalizeFingerprintMode(mode, fingerprint) + resolvedMode := fingerprints.NormalizeMode(mode, fingerprint) - if resolvedMode == FingerprintModeRegex { + if resolvedMode == fingerprints.FingerprintModeRegex { re, err := regexp.Compile(fingerprint) if err != nil { return false diff --git a/scanners/subdomain-takeover/internal/detect/body_matcher_test.go b/scanners/subdomain-takeover/internal/detect/body_matcher_test.go index a045a83a48..59df870656 100644 --- a/scanners/subdomain-takeover/internal/detect/body_matcher_test.go +++ b/scanners/subdomain-takeover/internal/detect/body_matcher_test.go @@ -6,11 +6,13 @@ import ( "strings" "testing" "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" ) func TestNoopBodyFingerprintMatcher(t *testing.T) { matcher := NewNoopBodyFingerprintMatcher() - if matcher.Contains("example.ca", "anything", FingerprintModeLiteral) { + if matcher.Contains("example.ca", "anything", fingerprints.FingerprintModeLiteral) { t.Fatal("noop matcher should never match") } } @@ -18,12 +20,12 @@ func TestNoopBodyFingerprintMatcher(t *testing.T) { func TestHTTPBodyFingerprintMatcher(t *testing.T) { t.Run("returns false when matcher/client nil", func(t *testing.T) { var matcher *HTTPBodyFingerprintMatcher - if matcher.Contains("example.ca", "x", FingerprintModeLiteral) { + if matcher.Contains("example.ca", "x", fingerprints.FingerprintModeLiteral) { t.Fatal("expected false for nil matcher") } matcher = &HTTPBodyFingerprintMatcher{} - if matcher.Contains("example.ca", "x", FingerprintModeLiteral) { + if matcher.Contains("example.ca", "x", fingerprints.FingerprintModeLiteral) { t.Fatal("expected false for nil client") } }) @@ -37,13 +39,13 @@ func TestHTTPBodyFingerprintMatcher(t *testing.T) { domain := strings.TrimPrefix(server.URL, "http://") matcher := NewHTTPBodyFingerprintMatcher(2 * time.Second) - if !matcher.Contains(domain, "sample app", FingerprintModeLiteral) { + if !matcher.Contains(domain, "sample app", fingerprints.FingerprintModeLiteral) { t.Fatal("expected literal substring match") } - if !matcher.Contains(domain, `Error Code: \d+`, FingerprintModeRegex) { + if !matcher.Contains(domain, `Error Code: \d+`, fingerprints.FingerprintModeRegex) { t.Fatal("expected regex match") } - if matcher.Contains(domain, "does-not-exist", FingerprintModeLiteral) { + if matcher.Contains(domain, "does-not-exist", fingerprints.FingerprintModeLiteral) { t.Fatal("did not expect missing literal to match") } }) @@ -56,7 +58,7 @@ func TestHTTPBodyFingerprintMatcher(t *testing.T) { domain := strings.TrimPrefix(server.URL, "http://") matcher := NewHTTPBodyFingerprintMatcher(2 * time.Second) - if matcher.Contains(domain, "(", FingerprintModeRegex) { + if matcher.Contains(domain, "(", fingerprints.FingerprintModeRegex) { t.Fatal("expected false for invalid regex") } }) diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index ffd1d527c6..34ccd35246 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -51,13 +51,13 @@ func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { func Classify(input model.Input, matcher BodyFingerprintMatcher, source FingerprintSource, logger zerolog.Logger) ([]model.Finding, error) { findings := []model.Finding{} - cnameFingerprints := source.CNAME() - nsFingerprints := source.NS() + cnameProviderFingerprints := source.CNAME() + nsProviderFingerprints := source.NS() cnameEvidence := ExtractCNAMEEvidence(input.Results) if cnameEvidence != nil { logger.Debug().Str("domain_key", input.DomainKey).Str("domain", cnameEvidence.Domain).Msg("cname evidence extracted") - cnameHit := MatchCNAMEFingerprints(*cnameEvidence, cnameFingerprints, matcher) + cnameHit := MatchCNAMEFingerprints(*cnameEvidence, cnameProviderFingerprints, matcher) if ShouldEmitCNAME(cnameHit) { logger.Debug(). Str("domain_key", input.DomainKey). @@ -94,7 +94,7 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, source Fingerpr Str("domain", nsEvidence.Domain). Int("ns_hosts", len(nsEvidence.NSHosts)). Msg("ns evidence extracted") - nsHit := MatchNSProviderRules(*nsEvidence, nsFingerprints) + nsHit := MatchNSProviderRules(*nsEvidence, nsProviderFingerprints) if ShouldEmitNSHijack(nsHit) { logger.Debug(). Str("domain_key", input.DomainKey). diff --git a/scanners/subdomain-takeover/internal/detect/classifier_test.go b/scanners/subdomain-takeover/internal/detect/classifier_test.go index 221c37ec25..62f19ec04a 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier_test.go +++ b/scanners/subdomain-takeover/internal/detect/classifier_test.go @@ -3,18 +3,19 @@ package detect import ( "testing" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" "github.com/rs/zerolog" ) func TestClassify_ExpectedBehavior(t *testing.T) { - cnameFPs := []CNAMEProviderFingerprint{ + cnameFPs := []fingerprints.CNAMEProviderFingerprint{ {Name: "Azure", Cname: []string{"azurewebsites.net"}, Nxdomain: true, Fingerprint: "unused"}, - {Name: "Ghost", Cname: []string{"ghost.io"}, Nxdomain: false, Fingerprint: "ghost 404", Mode: FingerprintModeLiteral}, + {Name: "Ghost", Cname: []string{"ghost.io"}, Nxdomain: false, Fingerprint: "ghost 404", Mode: fingerprints.FingerprintModeLiteral}, } - nsFPs := []NSProviderFingerprint{ - {Name: "RiskyDNS", Status: NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, - {Name: "SafeDNS", Status: NSStatusNotVulnerable, HostPatterns: []string{"*.safe-dns.net"}}, + nsFPs := []fingerprints.NSProviderFingerprint{ + {Name: "RiskyDNS", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, + {Name: "SafeDNS", Status: fingerprints.NSStatusNotVulnerable, HostPatterns: []string{"*.safe-dns.net"}}, } source := fakeSource{cname: cnameFPs, ns: nsFPs} @@ -77,7 +78,7 @@ func TestClassify_ExpectedBehavior(t *testing.T) { }, } - matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode FingerprintMode) bool { + matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { return true }} diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules.go b/scanners/subdomain-takeover/internal/detect/cname_rules.go index b693a4cd70..bc1023182e 100644 --- a/scanners/subdomain-takeover/internal/detect/cname_rules.go +++ b/scanners/subdomain-takeover/internal/detect/cname_rules.go @@ -1,5 +1,7 @@ package detect +import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + type CNAMEHit struct { Matched bool Provider string @@ -7,8 +9,8 @@ type CNAMEHit struct { NeedsNX bool } -func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { - for _, fp := range fingerprints { +func MatchCNAMEFingerprints(evidence CNAMEEvidence, providerFingerprints []fingerprints.CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { + for _, fp := range providerFingerprints { if fp.ContainsTarget(evidence.Target) { detectLogger.Debug(). Str("domain", evidence.Domain). @@ -32,7 +34,7 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, fingerprints []CNAMEProvider } } else { hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP - mode := normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + mode := fingerprints.NormalizeMode(fp.Mode, fp.Fingerprint) detectLogger.Debug(). Str("domain", evidence.Domain). Str("provider", fp.Name). diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules_test.go b/scanners/subdomain-takeover/internal/detect/cname_rules_test.go index d3a85d9006..a8c2e217c8 100644 --- a/scanners/subdomain-takeover/internal/detect/cname_rules_test.go +++ b/scanners/subdomain-takeover/internal/detect/cname_rules_test.go @@ -1,9 +1,13 @@ package detect -import "testing" +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" +) func TestMatchCNAMEFingerprints(t *testing.T) { - fps := []CNAMEProviderFingerprint{ + fps := []fingerprints.CNAMEProviderFingerprint{ { Name: "Azure", Cname: []string{"azurewebsites.net"}, @@ -15,7 +19,7 @@ func TestMatchCNAMEFingerprints(t *testing.T) { Cname: []string{"ghost.io"}, Nxdomain: false, Fingerprint: "The thing you were looking for is no longer here", - Mode: FingerprintModeLiteral, + Mode: fingerprints.FingerprintModeLiteral, }, } @@ -56,8 +60,8 @@ func TestMatchCNAMEFingerprints(t *testing.T) { t.Run("body fingerprint provider emits when matcher returns true", func(t *testing.T) { evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "blog.ghost.io", NoResolve: false} - matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode FingerprintMode) bool { - if domain != "a.example.ca" || mode != FingerprintModeLiteral { + matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { + if domain != "a.example.ca" || mode != fingerprints.FingerprintModeLiteral { t.Fatalf("unexpected matcher args: domain=%q mode=%q", domain, mode) } return true @@ -76,7 +80,7 @@ func TestMatchCNAMEFingerprints(t *testing.T) { t.Run("body fingerprint provider suppressed when matcher false", func(t *testing.T) { evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "blog.ghost.io", NoResolve: false} - got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{containsFn: func(string, string, FingerprintMode) bool { return false }}) + got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{containsFn: func(string, string, fingerprints.FingerprintMode) bool { return false }}) if got == nil { t.Fatal("expected hit, got nil") } diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_source.go b/scanners/subdomain-takeover/internal/detect/fingerprint_source.go index 2ecc9981f8..b6b202e1d7 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprint_source.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_source.go @@ -1,16 +1,18 @@ package detect +import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + type FingerprintSource interface { - CNAME() []CNAMEProviderFingerprint - NS() []NSProviderFingerprint + CNAME() []fingerprints.CNAMEProviderFingerprint + NS() []fingerprints.NSProviderFingerprint } type GlobalFingerprintSource struct{} -func (GlobalFingerprintSource) CNAME() []CNAMEProviderFingerprint { - return CNAMEProviderFingerprints +func (GlobalFingerprintSource) CNAME() []fingerprints.CNAMEProviderFingerprint { + return fingerprints.CNAME() } -func (GlobalFingerprintSource) NS() []NSProviderFingerprint { - return NSProviderFingerprints +func (GlobalFingerprintSource) NS() []fingerprints.NSProviderFingerprint { + return fingerprints.NS() } diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go b/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go index fa9c8160ec..34f45f1c3b 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go @@ -1,23 +1,22 @@ package detect -import "testing" +import ( + "testing" -func TestGlobalFingerprintSource(t *testing.T) { - origCNAME := CNAMEProviderFingerprints - origNS := NSProviderFingerprints - t.Cleanup(func() { - CNAMEProviderFingerprints = origCNAME - NSProviderFingerprints = origNS - }) + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/rs/zerolog" +) - CNAMEProviderFingerprints = []CNAMEProviderFingerprint{{Name: "A"}} - NSProviderFingerprints = []NSProviderFingerprint{{Name: "B"}} +func TestGlobalFingerprintSource(t *testing.T) { + if err := fingerprints.Load(zerolog.Nop()); err != nil { + t.Fatalf("failed to load fingerprints: %v", err) + } src := GlobalFingerprintSource{} - if got := src.CNAME(); len(got) != 1 || got[0].Name != "A" { - t.Fatalf("unexpected cname source: %+v", got) + if got := src.CNAME(); len(got) == 0 { + t.Fatal("expected cname source to be populated") } - if got := src.NS(); len(got) != 1 || got[0].Name != "B" { - t.Fatalf("unexpected ns source: %+v", got) + if got := src.NS(); len(got) == 0 { + t.Fatal("expected ns source to be populated") } } diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go index 953cee5580..a5b2485984 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_reasoning.go +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go @@ -1,8 +1,12 @@ package detect -import "strings" +import ( + "strings" -func getNSHijackReasonCode(lameType string, providerStatus NSProviderStatus) ReasonCode { + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" +) + +func getNSHijackReasonCode(lameType string, providerStatus fingerprints.NSProviderStatus) ReasonCode { switch normalizeLameType(lameType) { case "full": if isExploitableProviderStatus(providerStatus) { @@ -23,8 +27,8 @@ func isNSMatch(reasonCode ReasonCode) bool { return reasonCode == ReasonNSFullLameProviderVulnerable || reasonCode == ReasonNSPartialLameProviderVulnerable } -func isExploitableProviderStatus(status NSProviderStatus) bool { - return status == NSStatusVulnerable || status == NSStatusVulnerableWithPurchase +func isExploitableProviderStatus(status fingerprints.NSProviderStatus) bool { + return status == fingerprints.NSStatusVulnerable || status == fingerprints.NSStatusVulnerableWithPurchase } func normalizeLameType(lameType string) string { diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go index 3521974672..28b85a4ed4 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go @@ -1,21 +1,25 @@ package detect -import "testing" +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" +) func TestGetNSHijackReasonCode(t *testing.T) { tests := []struct { name string lameType string - status NSProviderStatus + status fingerprints.NSProviderStatus want ReasonCode }{ - {name: "full vulnerable", lameType: "full", status: NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, - {name: "partial vulnerable", lameType: "partial", status: NSStatusVulnerable, want: ReasonNSPartialLameProviderVulnerable}, - {name: "full vulnerable with purchase", lameType: "full", status: NSStatusVulnerableWithPurchase, want: ReasonNSFullLameProviderVulnerable}, - {name: "full not vulnerable", lameType: "full", status: NSStatusNotVulnerable, want: ReasonNSLameProviderUnknown}, - {name: "partial registration closed", lameType: "partial", status: NSStatusRegistrationClosed, want: ReasonNSLameProviderUnknown}, - {name: "unknown lame type", lameType: "none", status: NSStatusVulnerable, want: ReasonNSProviderMatchOnly}, - {name: "normalized lame type", lameType: " FULL ", status: NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, + {name: "full vulnerable", lameType: "full", status: fingerprints.NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, + {name: "partial vulnerable", lameType: "partial", status: fingerprints.NSStatusVulnerable, want: ReasonNSPartialLameProviderVulnerable}, + {name: "full vulnerable with purchase", lameType: "full", status: fingerprints.NSStatusVulnerableWithPurchase, want: ReasonNSFullLameProviderVulnerable}, + {name: "full not vulnerable", lameType: "full", status: fingerprints.NSStatusNotVulnerable, want: ReasonNSLameProviderUnknown}, + {name: "partial registration closed", lameType: "partial", status: fingerprints.NSStatusRegistrationClosed, want: ReasonNSLameProviderUnknown}, + {name: "unknown lame type", lameType: "none", status: fingerprints.NSStatusVulnerable, want: ReasonNSProviderMatchOnly}, + {name: "normalized lame type", lameType: " FULL ", status: fingerprints.NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, } for _, tt := range tests { @@ -29,13 +33,13 @@ func TestGetNSHijackReasonCode(t *testing.T) { } func TestNSReasoningHelpers(t *testing.T) { - if !isExploitableProviderStatus(NSStatusVulnerable) { + if !isExploitableProviderStatus(fingerprints.NSStatusVulnerable) { t.Fatal("expected vulnerable to be exploitable") } - if !isExploitableProviderStatus(NSStatusVulnerableWithPurchase) { + if !isExploitableProviderStatus(fingerprints.NSStatusVulnerableWithPurchase) { t.Fatal("expected vulnerable_with_purchase to be exploitable") } - if isExploitableProviderStatus(NSStatusNotVulnerable) { + if isExploitableProviderStatus(fingerprints.NSStatusNotVulnerable) { t.Fatal("expected not_vulnerable to be non-exploitable") } diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules.go b/scanners/subdomain-takeover/internal/detect/ns_rules.go index 5485e9cc95..8168cc415c 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_rules.go +++ b/scanners/subdomain-takeover/internal/detect/ns_rules.go @@ -1,15 +1,17 @@ package detect +import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + type NSHit struct { - Matched bool - Host string - Provider string - ReasonCode ReasonCode + Matched bool + Host string + Provider string + ReasonCode ReasonCode } -func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerprint) *NSHit { - if len(evidence.NSHosts) == 0 || len(fingerprints) == 0 { - detectLogger.Debug().Int("ns_hosts", len(evidence.NSHosts)).Int("fingerprints", len(fingerprints)).Msg("skipping ns matching due to insufficient inputs") +func MatchNSProviderRules(evidence NSEvidence, providerFingerprints []fingerprints.NSProviderFingerprint) *NSHit { + if len(evidence.NSHosts) == 0 || len(providerFingerprints) == 0 { + detectLogger.Debug().Int("ns_hosts", len(evidence.NSHosts)).Int("fingerprints", len(providerFingerprints)).Msg("skipping ns matching due to insufficient inputs") return nil } @@ -17,7 +19,7 @@ func MatchNSProviderRules(evidence NSEvidence, fingerprints []NSProviderFingerpr var best *NSHit for _, host := range evidence.NSHosts { - for _, fp := range fingerprints { + for _, fp := range providerFingerprints { if fp.ContainsNSHost(host) { reasonCode := getNSHijackReasonCode(lameType, fp.Status) rank := nsReasonRank(reasonCode) diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go index c12671323d..858a60e15b 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go +++ b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go @@ -3,13 +3,14 @@ package detect import ( "testing" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) func TestMatchNSProviderRules(t *testing.T) { - nsFP := []NSProviderFingerprint{ - {Name: "UnknownDNS", Status: NSStatusNotVulnerable, HostPatterns: []string{"*.unknown-dns.net"}}, - {Name: "RiskyDNS", Status: NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, + nsFP := []fingerprints.NSProviderFingerprint{ + {Name: "UnknownDNS", Status: fingerprints.NSStatusNotVulnerable, HostPatterns: []string{"*.unknown-dns.net"}}, + {Name: "RiskyDNS", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, } t.Run("returns nil for missing hosts", func(t *testing.T) { diff --git a/scanners/subdomain-takeover/internal/detect/test_helpers_test.go b/scanners/subdomain-takeover/internal/detect/test_helpers_test.go index f73fcca80f..057f7101eb 100644 --- a/scanners/subdomain-takeover/internal/detect/test_helpers_test.go +++ b/scanners/subdomain-takeover/internal/detect/test_helpers_test.go @@ -1,14 +1,15 @@ package detect import ( + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) type fakeMatcher struct { - containsFn func(domain string, fingerprint string, mode FingerprintMode) bool + containsFn func(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool } -func (f fakeMatcher) Contains(domain string, fingerprint string, mode FingerprintMode) bool { +func (f fakeMatcher) Contains(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { if f.containsFn == nil { return false } @@ -16,12 +17,12 @@ func (f fakeMatcher) Contains(domain string, fingerprint string, mode Fingerprin } type fakeSource struct { - cname []CNAMEProviderFingerprint - ns []NSProviderFingerprint + cname []fingerprints.CNAMEProviderFingerprint + ns []fingerprints.NSProviderFingerprint } -func (f fakeSource) CNAME() []CNAMEProviderFingerprint { return f.cname } -func (f fakeSource) NS() []NSProviderFingerprint { return f.ns } +func (f fakeSource) CNAME() []fingerprints.CNAMEProviderFingerprint { return f.cname } +func (f fakeSource) NS() []fingerprints.NSProviderFingerprint { return f.ns } func strPtr(v string) *string { return &v } diff --git a/scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json b/scanners/subdomain-takeover/internal/fingerprints/data/cname_fingerprints.json similarity index 100% rename from scanners/subdomain-takeover/internal/detect/data/cname_fingerprints.json rename to scanners/subdomain-takeover/internal/fingerprints/data/cname_fingerprints.json diff --git a/scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json b/scanners/subdomain-takeover/internal/fingerprints/data/ns_fingerprints.json similarity index 100% rename from scanners/subdomain-takeover/internal/detect/data/ns_fingerprints.json rename to scanners/subdomain-takeover/internal/fingerprints/data/ns_fingerprints.json diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go b/scanners/subdomain-takeover/internal/fingerprints/loader.go similarity index 79% rename from scanners/subdomain-takeover/internal/detect/fingerprint_loader.go rename to scanners/subdomain-takeover/internal/fingerprints/loader.go index da2101e51a..9bcec10792 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprint_loader.go +++ b/scanners/subdomain-takeover/internal/fingerprints/loader.go @@ -1,4 +1,4 @@ -package detect +package fingerprints import ( "embed" @@ -11,55 +11,66 @@ import ( ) //go:embed data/*.json -var fingerprintFS embed.FS +var dataFS embed.FS var ( - loadFingerprintsOnce sync.Once - loadFingerprintsErr error + loadOnce sync.Once + loadErr error + + cnameProviderFingerprints []CNAMEProviderFingerprint + nsProviderFingerprints []NSProviderFingerprint ) -func LoadFingerprints(logger zerolog.Logger) error { +func Load(logger zerolog.Logger) error { log := logger.With().Str("component", "fingerprint_loader").Logger() - loadFingerprintsOnce.Do(func() { + loadOnce.Do(func() { var cname []CNAMEProviderFingerprint var ns []NSProviderFingerprint if err := loadJSON("data/cname_fingerprints.json", &cname); err != nil { log.Error().Err(err).Str("dataset", "cname_fingerprints").Msg("failed to load fingerprint dataset") - loadFingerprintsErr = err + loadErr = err return } if err := loadJSON("data/ns_fingerprints.json", &ns); err != nil { log.Error().Err(err).Str("dataset", "ns_fingerprints").Msg("failed to load fingerprint dataset") - loadFingerprintsErr = err + loadErr = err return } if err := validateCNAMEFingerprints(cname); err != nil { log.Error().Err(err).Str("dataset", "cname_fingerprints").Msg("invalid fingerprint dataset") - loadFingerprintsErr = err + loadErr = err return } if err := validateNSFingerprints(ns); err != nil { log.Error().Err(err).Str("dataset", "ns_fingerprints").Msg("invalid fingerprint dataset") - loadFingerprintsErr = err + loadErr = err return } - CNAMEProviderFingerprints = cname - NSProviderFingerprints = ns + cnameProviderFingerprints = cname + nsProviderFingerprints = ns log.Info().Int("cname_fingerprints", len(cname)).Int("ns_fingerprints", len(ns)).Msg("fingerprint datasets loaded") }) - return loadFingerprintsErr + return loadErr +} + +func CNAME() []CNAMEProviderFingerprint { + return cnameProviderFingerprints +} + +func NS() []NSProviderFingerprint { + return nsProviderFingerprints } func loadJSON(path string, out any) error { - b, err := fingerprintFS.ReadFile(path) + b, err := dataFS.ReadFile(path) if err != nil { return fmt.Errorf("read %s: %w", path, err) } @@ -90,7 +101,7 @@ func validateCNAMEFingerprints(fingerprints []CNAMEProviderFingerprint) error { return fmt.Errorf("cname fingerprint[%d] missing fingerprint", i) } - fingerprints[i].Mode = normalizeFingerprintMode(fp.Mode, fp.Fingerprint) + fingerprints[i].Mode = NormalizeMode(fp.Mode, fp.Fingerprint) } return nil diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_loader_test.go b/scanners/subdomain-takeover/internal/fingerprints/loader_test.go similarity index 50% rename from scanners/subdomain-takeover/internal/detect/fingerprint_loader_test.go rename to scanners/subdomain-takeover/internal/fingerprints/loader_test.go index 726d95b4d9..68dff54530 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprint_loader_test.go +++ b/scanners/subdomain-takeover/internal/fingerprints/loader_test.go @@ -1,4 +1,4 @@ -package detect +package fingerprints import ( "sync" @@ -7,32 +7,31 @@ import ( "github.com/rs/zerolog" ) -func TestLoadFingerprints(t *testing.T) { - origOnce := loadFingerprintsOnce - origErr := loadFingerprintsErr - origCNAME := CNAMEProviderFingerprints - origNS := NSProviderFingerprints +func TestLoad(t *testing.T) { + origErr := loadErr + origCNAME := cnameProviderFingerprints + origNS := nsProviderFingerprints - loadFingerprintsOnce = sync.Once{} - loadFingerprintsErr = nil - CNAMEProviderFingerprints = nil - NSProviderFingerprints = nil + loadOnce = sync.Once{} + loadErr = nil + cnameProviderFingerprints = nil + nsProviderFingerprints = nil t.Cleanup(func() { - loadFingerprintsOnce = origOnce - loadFingerprintsErr = origErr - CNAMEProviderFingerprints = origCNAME - NSProviderFingerprints = origNS + loadOnce = sync.Once{} + loadErr = origErr + cnameProviderFingerprints = origCNAME + nsProviderFingerprints = origNS }) - err := LoadFingerprints(zerolog.Nop()) + err := Load(zerolog.Nop()) if err != nil { - t.Fatalf("LoadFingerprints returned error: %v", err) + t.Fatalf("Load returned error: %v", err) } - if len(CNAMEProviderFingerprints) == 0 { + if len(CNAME()) == 0 { t.Fatal("expected cname fingerprints to be loaded") } - if len(NSProviderFingerprints) == 0 { + if len(NS()) == 0 { t.Fatal("expected ns fingerprints to be loaded") } } diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints.go b/scanners/subdomain-takeover/internal/fingerprints/types.go similarity index 92% rename from scanners/subdomain-takeover/internal/detect/fingerprints.go rename to scanners/subdomain-takeover/internal/fingerprints/types.go index d1491995cd..ac275d8430 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprints.go +++ b/scanners/subdomain-takeover/internal/fingerprints/types.go @@ -1,4 +1,4 @@ -package detect +package fingerprints import "strings" @@ -17,9 +17,6 @@ const ( FingerprintModeRegex FingerprintMode = "regex" ) -var CNAMEProviderFingerprints []CNAMEProviderFingerprint -var NSProviderFingerprints []NSProviderFingerprint - type NSProviderStatus string const ( @@ -95,7 +92,7 @@ func (f *CNAMEProviderFingerprint) ContainsTarget(target string) bool { return false } -func normalizeFingerprintMode(mode FingerprintMode, fingerprint string) FingerprintMode { +func NormalizeMode(mode FingerprintMode, fingerprint string) FingerprintMode { if mode == FingerprintModeLiteral || mode == FingerprintModeRegex { return mode } diff --git a/scanners/subdomain-takeover/internal/detect/fingerprints_test.go b/scanners/subdomain-takeover/internal/fingerprints/types_test.go similarity index 75% rename from scanners/subdomain-takeover/internal/detect/fingerprints_test.go rename to scanners/subdomain-takeover/internal/fingerprints/types_test.go index a31934890e..96be0d1183 100644 --- a/scanners/subdomain-takeover/internal/detect/fingerprints_test.go +++ b/scanners/subdomain-takeover/internal/fingerprints/types_test.go @@ -1,4 +1,4 @@ -package detect +package fingerprints import "testing" @@ -32,16 +32,16 @@ func TestContainsTargetAndNormalizeMode(t *testing.T) { t.Fatal("did not expect suffix mismatch") } - if got := normalizeFingerprintMode(FingerprintModeLiteral, "foo"); got != FingerprintModeLiteral { + if got := NormalizeMode(FingerprintModeLiteral, "foo"); got != FingerprintModeLiteral { t.Fatalf("unexpected mode: %q", got) } - if got := normalizeFingerprintMode("", "service unavailable"); got != FingerprintModeLiteral { + if got := NormalizeMode("", "service unavailable"); got != FingerprintModeLiteral { t.Fatalf("unexpected inferred mode: %q", got) } - if got := normalizeFingerprintMode("", "Error: .* not found"); got != FingerprintModeRegex { + if got := NormalizeMode("", "Error: .* not found"); got != FingerprintModeRegex { t.Fatalf("unexpected inferred mode: %q", got) } - if got := normalizeFingerprintMode("", "foo\\d+"); got != FingerprintModeRegex { + if got := NormalizeMode("", "foo\\d+"); got != FingerprintModeRegex { t.Fatalf("unexpected inferred mode: %q", got) } } From ce87343ebc87a780f73e52cb3d0f05c7fb243ecc Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Thu, 23 Jul 2026 17:13:07 -0300 Subject: [PATCH 40/65] add Makefile for local development --- scanners/subdomain-takeover/Makefile | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 scanners/subdomain-takeover/Makefile diff --git a/scanners/subdomain-takeover/Makefile b/scanners/subdomain-takeover/Makefile new file mode 100644 index 0000000000..72345ab0b2 --- /dev/null +++ b/scanners/subdomain-takeover/Makefile @@ -0,0 +1,47 @@ +.PHONY: help run test test-race build fmt fmt-check vet lint tidy ci + +GO ?= go +SERVICE_BIN ?= subdomain-takeover +BUILD_DIR ?= bin + +help: + @printf "Targets:\n" + @printf " make run - Run the service\n" + @printf " make test - Run all tests\n" + @printf " make test-race - Run tests with race detector\n" + @printf " make build - Build service binary\n" + @printf " make fmt - Format Go files\n" + @printf " make fmt-check - Check formatting (no changes)\n" + @printf " make vet - Run go vet\n" + @printf " make lint - Run fmt-check + vet\n" + @printf " make tidy - Tidy modules\n" + @printf " make ci - Lint, test, build\n" + +run: + $(GO) run ./cmd/service + +test: + $(GO) test ./... + +test-race: + $(GO) test -race ./... + +build: + mkdir -p $(BUILD_DIR) + CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(SERVICE_BIN) ./cmd/service + +fmt: + $(GO) fmt ./... + +fmt-check: + @test -z "$$($(GO)fmt -l .)" || (printf "Unformatted files found. Run 'make fmt'.\n" && exit 1) + +vet: + $(GO) vet ./... + +lint: fmt-check vet + +tidy: + $(GO) mod tidy + +ci: lint test build From 6ecde7dfc0d461bc957f07da7d5a6e9e82b5383a Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Thu, 23 Jul 2026 17:13:15 -0300 Subject: [PATCH 41/65] add cloudbuild.yaml --- scanners/subdomain-takeover/cloudbuild.yaml | 43 +++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 scanners/subdomain-takeover/cloudbuild.yaml diff --git a/scanners/subdomain-takeover/cloudbuild.yaml b/scanners/subdomain-takeover/cloudbuild.yaml new file mode 100644 index 0000000000..64bd3d7902 --- /dev/null +++ b/scanners/subdomain-takeover/cloudbuild.yaml @@ -0,0 +1,43 @@ +steps: + - name: "golang:1.25" + id: ci-checks + dir: scanners/subdomain-takeover + entrypoint: "bash" + args: + - "-c" + - | + make ci + + - name: "gcr.io/cloud-builders/docker" + id: generate-image-name + entrypoint: "bash" + dir: scanners/subdomain-takeover + args: + - "-c" + - | + echo "northamerica-northeast1-docker.pkg.dev/track-compliance/tracker/subdomain-takeover:$(echo $BRANCH_NAME | sed 's/[^a-zA-Z0-9]/-/g')-$SHORT_SHA-$(date +%s)" > /workspace/imagename + + - name: "gcr.io/cloud-builders/docker" + id: build-scanner + entrypoint: "bash" + dir: scanners/subdomain-takeover + args: + - "-c" + - | + image=$(cat /workspace/imagename) + docker build -t $image . + + - name: "gcr.io/cloud-builders/docker" + id: push-scanner-if-master + entrypoint: "bash" + dir: scanners/subdomain-takeover + args: + - "-c" + - | + if [[ "$BRANCH_NAME" == "master" ]] + then + image=$(cat /workspace/imagename) + docker push $image + else + exit 0 + fi From 763e4f70ab36cb9345a629b73151f9aa3257b121 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 24 Jul 2026 17:49:39 -0300 Subject: [PATCH 42/65] add registrar mismatch rules to NS hijack detection logic --- .../subdomain-takeover/internal/app/runner.go | 4 +- .../internal/app/runner_test.go | 47 +++++- .../internal/detect/classifier_test.go | 8 +- .../internal/detect/confidence.go | 4 + .../internal/detect/ns_evidence.go | 3 +- .../internal/detect/ns_reasoning.go | 70 +++++++-- .../internal/detect/ns_reasoning_test.go | 95 +++++++++--- .../internal/detect/ns_rules.go | 14 +- .../internal/detect/ns_rules_test.go | 55 ++++++- .../internal/detect/reason_codes.go | 2 + .../internal/detect/registrar_helpers.go | 139 ++++++++++++++++++ .../internal/fingerprints/loader_test.go | 48 ++++++ .../internal/messaging/health.go | 11 ++ .../internal/messaging/health_test.go | 41 +++++- .../internal/messaging/publisher.go | 4 +- .../internal/messaging/publisher_test.go | 20 +++ 16 files changed, 516 insertions(+), 49 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/detect/registrar_helpers.go diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index b8c0121f87..924427b855 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -23,6 +23,8 @@ type MessageHandler interface { Handle(ctx context.Context, msg jetstream.Msg) error } +var checkConnection = messaging.CheckConnection + func Run(ctx context.Context, deps RunnerDeps) { logger := deps.Logger iter := deps.Iter @@ -39,7 +41,7 @@ func Run(ctx context.Context, deps RunnerDeps) { Loop: for { - if err := messaging.CheckConnection(deps.NC); err != nil { + if err := checkConnection(deps.NC); err != nil { logger.Error().Err(err).Msg("NATS connection unhealthy") break Loop } diff --git a/scanners/subdomain-takeover/internal/app/runner_test.go b/scanners/subdomain-takeover/internal/app/runner_test.go index 7d076aa67d..b732ecd76a 100644 --- a/scanners/subdomain-takeover/internal/app/runner_test.go +++ b/scanners/subdomain-takeover/internal/app/runner_test.go @@ -2,8 +2,10 @@ package app import ( "context" + "errors" "sync" "testing" + "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -69,7 +71,19 @@ func TestRun_ReturnsImmediatelyWhenConnectionUnhealthy(t *testing.T) { } func TestRun_ClampsWorkerCountBelowOne(t *testing.T) { - iter := &fakeMessagesIter{} + origCheckConnection := checkConnection + t.Cleanup(func() { checkConnection = origCheckConnection }) + + checkCalls := 0 + checkConnection = func(_ *nats.Conn) error { + checkCalls++ + if checkCalls > 1 { + return errors.New("stop") + } + return nil + } + + iter := &fakeMessagesIter{msgs: []jetstream.Msg{&fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"}}} h := &fakeHandler{} deps := RunnerDeps{ @@ -77,10 +91,39 @@ func TestRun_ClampsWorkerCountBelowOne(t *testing.T) { WorkerCount: 0, Iter: iter, Worker: h, - NC: (*nats.Conn)(nil), + NC: nil, } Run(context.Background(), deps) + if h.count != 1 { + t.Fatalf("expected one handled message, got %d", h.count) + } +} + +func TestRun_ExitsWhenContextCancelledDuringNextErrors(t *testing.T) { + origCheckConnection := checkConnection + t.Cleanup(func() { checkConnection = origCheckConnection }) + checkConnection = func(_ *nats.Conn) error { return nil } + + iter := &fakeMessagesIter{err: errors.New("next failed")} + h := &fakeHandler{} + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(5 * time.Millisecond) + cancel() + }() + + deps := RunnerDeps{ + Logger: zerolog.Nop(), + WorkerCount: 1, + Iter: iter, + Worker: h, + NC: nil, + } + + Run(ctx, deps) + if h.count != 0 { t.Fatalf("expected no handled messages, got %d", h.count) } diff --git a/scanners/subdomain-takeover/internal/detect/classifier_test.go b/scanners/subdomain-takeover/internal/detect/classifier_test.go index 62f19ec04a..e09a055c7d 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier_test.go +++ b/scanners/subdomain-takeover/internal/detect/classifier_test.go @@ -14,18 +14,24 @@ func TestClassify_ExpectedBehavior(t *testing.T) { {Name: "Ghost", Cname: []string{"ghost.io"}, Nxdomain: false, Fingerprint: "ghost 404", Mode: fingerprints.FingerprintModeLiteral}, } nsFPs := []fingerprints.NSProviderFingerprint{ - {Name: "RiskyDNS", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, + {Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, {Name: "SafeDNS", Status: fingerprints.NSStatusNotVulnerable, HostPatterns: []string{"*.safe-dns.net"}}, } source := fakeSource{cname: cnameFPs, ns: nsFPs} t.Run("emits cname and ns findings when both are exploitable", func(t *testing.T) { + rdapMatch := true input := model.Input{ DomainKey: "k1", Results: model.ScanResults{ Domain: strPtr("a.example.ca"), CnameRecord: strPtr("a.example.ca. 300 IN CNAME foo.azurewebsites.net."), + RegistrarContext: &model.RegistrarContext{ + LookupSuccess: true, + RegistrarName: "Namecheap", + DelegationMatchesRDAP: &rdapMatch, + }, NsDelegations: &model.NsDelegations{ Hosts: []string{"ns1.risky-dns.net"}, Delegation: model.Delegation{ diff --git a/scanners/subdomain-takeover/internal/detect/confidence.go b/scanners/subdomain-takeover/internal/detect/confidence.go index 9865a561db..06fcd68e53 100644 --- a/scanners/subdomain-takeover/internal/detect/confidence.go +++ b/scanners/subdomain-takeover/internal/detect/confidence.go @@ -23,6 +23,10 @@ func ConfidenceForReason(reasonCode ReasonCode) string { return ConfidenceConfirmed case ReasonNSPartialLameProviderVulnerable: return ConfidenceProbable + case ReasonNSRegistrarProviderMatch: + return ConfidenceSuspected + case ReasonNSRegistrarContextInsufficient: + return ConfidenceSuspected case ReasonNSLameProviderUnknown: return ConfidenceSuspected case ReasonNSProviderMatchOnly: diff --git a/scanners/subdomain-takeover/internal/detect/ns_evidence.go b/scanners/subdomain-takeover/internal/detect/ns_evidence.go index 78ddb2f428..7115fdbe2d 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_evidence.go +++ b/scanners/subdomain-takeover/internal/detect/ns_evidence.go @@ -11,7 +11,7 @@ type NSEvidence struct { Domain string NSHosts []string NSDelegations model.NsDelegations - // Registrar model.RegistrarContext // if/when added + Registrar *model.RegistrarContext } func ExtractNSEvidence(input model.ScanResults) *NSEvidence { @@ -24,6 +24,7 @@ func ExtractNSEvidence(input model.ScanResults) *NSEvidence { Domain: *input.Domain, NSHosts: parseHostnames(input.NsDelegations.Hosts), NSDelegations: *input.NsDelegations, + Registrar: input.RegistrarContext, } } diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go index a5b2485984..bc0a672aca 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_reasoning.go +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go @@ -4,33 +4,73 @@ import ( "strings" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) -func getNSHijackReasonCode(lameType string, providerStatus fingerprints.NSProviderStatus) ReasonCode { +func getNSHijackReasonCode(lameType string, fingerprint fingerprints.NSProviderFingerprint, registrar *model.RegistrarContext) ReasonCode { + if !isLameType(lameType) { + return ReasonNSProviderMatchOnly + } + + if !isExploitableProviderStatus(fingerprint.Status) { + return ReasonNSLameProviderUnknown + } + + registrarReason := getRegistrarReasonCode(fingerprint.Name, registrar) + if registrarReason != "" { + return registrarReason + } + switch normalizeLameType(lameType) { case "full": - if isExploitableProviderStatus(providerStatus) { - return ReasonNSFullLameProviderVulnerable - } - return ReasonNSLameProviderUnknown + return ReasonNSFullLameProviderVulnerable case "partial": - if isExploitableProviderStatus(providerStatus) { - return ReasonNSPartialLameProviderVulnerable - } - return ReasonNSLameProviderUnknown + return ReasonNSPartialLameProviderVulnerable default: return ReasonNSProviderMatchOnly } } -func isNSMatch(reasonCode ReasonCode) bool { - return reasonCode == ReasonNSFullLameProviderVulnerable || reasonCode == ReasonNSPartialLameProviderVulnerable +func getRegistrarReasonCode(provider string, registrar *model.RegistrarContext) ReasonCode { + if registrar == nil { + return ReasonNSRegistrarContextInsufficient + } + + if !registrar.LookupSuccess || registrar.RegistrarName == "" { + return ReasonNSRegistrarContextInsufficient + } + + if registrar.DelegationMatchesRDAP != nil && !*registrar.DelegationMatchesRDAP { + return ReasonNSRegistrarContextInsufficient + } + + registrarKey := canonicalProviderKey(registrar.RegistrarName) + providerKey := canonicalProviderKey(provider) + + if registrarKey == "" || providerKey == "" { + return ReasonNSRegistrarContextInsufficient + } + + if registrarKey == providerKey { + return ReasonNSRegistrarProviderMatch + } + + return "" +} + +func isRegistrarMismatch(provider string, registrar *model.RegistrarContext) bool { + return getRegistrarReasonCode(provider, registrar) == "" } func isExploitableProviderStatus(status fingerprints.NSProviderStatus) bool { return status == fingerprints.NSStatusVulnerable || status == fingerprints.NSStatusVulnerableWithPurchase } +func isLameType(lameType string) bool { + ltNorm := normalizeLameType(lameType) + return ltNorm == "full" || ltNorm == "partial" +} + func normalizeLameType(lameType string) string { return strings.ToLower(strings.TrimSpace(lameType)) } @@ -41,10 +81,14 @@ func nsReasonRank(reasonCode ReasonCode) int { return 4 case ReasonNSPartialLameProviderVulnerable: return 3 - case ReasonNSLameProviderUnknown: + case ReasonNSRegistrarProviderMatch: return 2 - case ReasonNSProviderMatchOnly: + case ReasonNSRegistrarContextInsufficient: + return 2 + case ReasonNSLameProviderUnknown: return 1 + case ReasonNSProviderMatchOnly: + return 0 default: return 0 } diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go index 28b85a4ed4..3b8fccb923 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go @@ -4,27 +4,82 @@ import ( "testing" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" ) func TestGetNSHijackReasonCode(t *testing.T) { + namecheapRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"} + digitalOceanRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."} + invalidRegistrar := &model.RegistrarContext{LookupSuccess: false, RegistrarName: "Namecheap"} + tests := []struct { - name string - lameType string - status fingerprints.NSProviderStatus - want ReasonCode + name string + lameType string + provider fingerprints.NSProviderFingerprint + registrar *model.RegistrarContext + want ReasonCode }{ - {name: "full vulnerable", lameType: "full", status: fingerprints.NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, - {name: "partial vulnerable", lameType: "partial", status: fingerprints.NSStatusVulnerable, want: ReasonNSPartialLameProviderVulnerable}, - {name: "full vulnerable with purchase", lameType: "full", status: fingerprints.NSStatusVulnerableWithPurchase, want: ReasonNSFullLameProviderVulnerable}, - {name: "full not vulnerable", lameType: "full", status: fingerprints.NSStatusNotVulnerable, want: ReasonNSLameProviderUnknown}, - {name: "partial registration closed", lameType: "partial", status: fingerprints.NSStatusRegistrationClosed, want: ReasonNSLameProviderUnknown}, - {name: "unknown lame type", lameType: "none", status: fingerprints.NSStatusVulnerable, want: ReasonNSProviderMatchOnly}, - {name: "normalized lame type", lameType: " FULL ", status: fingerprints.NSStatusVulnerable, want: ReasonNSFullLameProviderVulnerable}, + { + name: "full vulnerable with registrar mismatch", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSFullLameProviderVulnerable, + }, + { + name: "partial vulnerable with registrar mismatch", + lameType: "partial", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSPartialLameProviderVulnerable, + }, + { + name: "full vulnerable with purchase and mismatch", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerableWithPurchase}, + registrar: namecheapRegistrar, + want: ReasonNSFullLameProviderVulnerable, + }, + { + name: "full non vulnerable", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusNotVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSLameProviderUnknown, + }, + { + name: "unknown lame type", + lameType: "none", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSProviderMatchOnly, + }, + { + name: "same registrar/provider suppressed", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: digitalOceanRegistrar, + want: ReasonNSRegistrarProviderMatch, + }, + { + name: "missing registrar context suppressed", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: nil, + want: ReasonNSRegistrarContextInsufficient, + }, + { + name: "invalid registrar context suppressed", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: invalidRegistrar, + want: ReasonNSRegistrarContextInsufficient, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := getNSHijackReasonCode(tt.lameType, tt.status) + got := getNSHijackReasonCode(tt.lameType, tt.provider, tt.registrar) if got != tt.want { t.Fatalf("getNSHijackReasonCode()=%q want=%q", got, tt.want) } @@ -47,20 +102,20 @@ func TestNSReasoningHelpers(t *testing.T) { t.Fatalf("normalizeLameType mismatch: %q", got) } - if !isNSMatch(ReasonNSFullLameProviderVulnerable) || !isNSMatch(ReasonNSPartialLameProviderVulnerable) { - t.Fatal("expected vulnerable reason codes to be emittable") + if !isRegistrarMismatch("Digital Ocean", &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"}) { + t.Fatal("expected registrar mismatch for different providers") } - if isNSMatch(ReasonNSLameProviderUnknown) { - t.Fatal("expected unknown reason not to be emittable") + if isRegistrarMismatch("Digital Ocean", &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."}) { + t.Fatal("expected same provider to fail mismatch check") } if nsReasonRank(ReasonNSFullLameProviderVulnerable) <= nsReasonRank(ReasonNSPartialLameProviderVulnerable) { t.Fatal("expected full lame rank > partial lame rank") } - if nsReasonRank(ReasonNSPartialLameProviderVulnerable) <= nsReasonRank(ReasonNSLameProviderUnknown) { - t.Fatal("expected partial lame rank > unknown rank") + if nsReasonRank(ReasonNSPartialLameProviderVulnerable) <= nsReasonRank(ReasonNSRegistrarProviderMatch) { + t.Fatal("expected partial lame rank > registrar-provider-match rank") } - if nsReasonRank(ReasonNSLameProviderUnknown) <= nsReasonRank(ReasonNSProviderMatchOnly) { - t.Fatal("expected unknown rank > provider-only rank") + if nsReasonRank(ReasonNSRegistrarProviderMatch) <= nsReasonRank(ReasonNSProviderMatchOnly) { + t.Fatal("expected registrar gate rank > provider-only rank") } } diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules.go b/scanners/subdomain-takeover/internal/detect/ns_rules.go index 8168cc415c..32de3049e3 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_rules.go +++ b/scanners/subdomain-takeover/internal/detect/ns_rules.go @@ -21,27 +21,29 @@ func MatchNSProviderRules(evidence NSEvidence, providerFingerprints []fingerprin for _, host := range evidence.NSHosts { for _, fp := range providerFingerprints { if fp.ContainsNSHost(host) { - reasonCode := getNSHijackReasonCode(lameType, fp.Status) - rank := nsReasonRank(reasonCode) hit := &NSHit{ - Matched: isNSMatch(reasonCode), + Matched: false, Host: host, Provider: fp.Name, - ReasonCode: reasonCode, + ReasonCode: getNSHijackReasonCode(lameType, fp, evidence.Registrar), } + hit.Matched = hit.ReasonCode == ReasonNSFullLameProviderVulnerable || hit.ReasonCode == ReasonNSPartialLameProviderVulnerable + + rank := nsReasonRank(hit.ReasonCode) + detectLogger.Debug(). Str("domain", evidence.Domain). Str("host", host). Str("provider", fp.Name). Str("provider_status", string(fp.Status)). Str("lame_type", lameType). - Str("reason_code", string(reasonCode)). + Str("reason_code", string(hit.ReasonCode)). Int("rank", rank). Bool("emittable", hit.Matched). Msg("ns candidate evaluated") - if best == nil || nsReasonRank(hit.ReasonCode) > nsReasonRank(best.ReasonCode) { + if best == nil || rank > nsReasonRank(best.ReasonCode) { best = hit } } diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go index 858a60e15b..497dc4160f 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go +++ b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go @@ -10,9 +10,12 @@ import ( func TestMatchNSProviderRules(t *testing.T) { nsFP := []fingerprints.NSProviderFingerprint{ {Name: "UnknownDNS", Status: fingerprints.NSStatusNotVulnerable, HostPatterns: []string{"*.unknown-dns.net"}}, - {Name: "RiskyDNS", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, + {Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, } + namecheapRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"} + digitalOceanRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."} + t.Run("returns nil for missing hosts", func(t *testing.T) { evidence := NSEvidence{Domain: "a.example.ca"} if got := MatchNSProviderRules(evidence, nsFP); got != nil { @@ -38,6 +41,7 @@ func TestMatchNSProviderRules(t *testing.T) { evidence := NSEvidence{ Domain: "a.example.ca", NSHosts: []string{"ns1.risky-dns.net"}, + Registrar: namecheapRegistrar, NSDelegations: model.NsDelegations{Delegation: model.Delegation{ LameType: "full", }}, @@ -46,7 +50,7 @@ func TestMatchNSProviderRules(t *testing.T) { if got == nil { t.Fatal("expected hit, got nil") } - if got.Provider != "RiskyDNS" { + if got.Provider != "Digital Ocean" { t.Fatalf("unexpected provider: %q", got.Provider) } if got.ReasonCode != ReasonNSFullLameProviderVulnerable { @@ -61,6 +65,7 @@ func TestMatchNSProviderRules(t *testing.T) { evidence := NSEvidence{ Domain: "a.example.ca", NSHosts: []string{"ns1.unknown-dns.net", "ns1.risky-dns.net"}, + Registrar: namecheapRegistrar, NSDelegations: model.NsDelegations{Delegation: model.Delegation{ LameType: "partial", }}, @@ -69,7 +74,7 @@ func TestMatchNSProviderRules(t *testing.T) { if got == nil { t.Fatal("expected hit, got nil") } - if got.Provider != "RiskyDNS" { + if got.Provider != "Digital Ocean" { t.Fatalf("expected risky provider to win ranking, got %q", got.Provider) } if got.ReasonCode != ReasonNSPartialLameProviderVulnerable { @@ -81,6 +86,7 @@ func TestMatchNSProviderRules(t *testing.T) { evidence := NSEvidence{ Domain: "a.example.ca", NSHosts: []string{"ns1.unknown-dns.net"}, + Registrar: namecheapRegistrar, NSDelegations: model.NsDelegations{Delegation: model.Delegation{ LameType: "full", }}, @@ -96,6 +102,49 @@ func TestMatchNSProviderRules(t *testing.T) { t.Fatal("expected matched=false for non-vulnerable state") } }) + + t.Run("suppresses vulnerable provider when registrar matches provider", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.risky-dns.net"}, + Registrar: digitalOceanRegistrar, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + + got := MatchNSProviderRules(evidence, nsFP) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.ReasonCode != ReasonNSRegistrarProviderMatch { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if got.Matched { + t.Fatal("expected matched=false when registrar and provider match") + } + }) + + t.Run("suppresses vulnerable provider when registrar context missing", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.risky-dns.net"}, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + + got := MatchNSProviderRules(evidence, nsFP) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.ReasonCode != ReasonNSRegistrarContextInsufficient { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if got.Matched { + t.Fatal("expected matched=false when registrar context is missing") + } + }) } func TestShouldEmitNSHijack(t *testing.T) { diff --git a/scanners/subdomain-takeover/internal/detect/reason_codes.go b/scanners/subdomain-takeover/internal/detect/reason_codes.go index f080b18329..6a79589ded 100644 --- a/scanners/subdomain-takeover/internal/detect/reason_codes.go +++ b/scanners/subdomain-takeover/internal/detect/reason_codes.go @@ -10,6 +10,8 @@ const ( ReasonNSFullLameProviderVulnerable ReasonCode = "NS_FULL_LAME_PROVIDER_VULNERABLE" ReasonNSPartialLameProviderVulnerable ReasonCode = "NS_PARTIAL_LAME_PROVIDER_VULNERABLE" + ReasonNSRegistrarProviderMatch ReasonCode = "NS_REGISTRAR_PROVIDER_MATCH" + ReasonNSRegistrarContextInsufficient ReasonCode = "NS_REGISTRAR_CONTEXT_INSUFFICIENT" ReasonNSLameProviderUnknown ReasonCode = "NS_LAME_PROVIDER_UNKNOWN" ReasonNSProviderMatchOnly ReasonCode = "NS_PROVIDER_MATCH_ONLY" ) diff --git a/scanners/subdomain-takeover/internal/detect/registrar_helpers.go b/scanners/subdomain-takeover/internal/detect/registrar_helpers.go new file mode 100644 index 0000000000..a31f942744 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/registrar_helpers.go @@ -0,0 +1,139 @@ +package detect + +import "strings" + +func normalizeOrgName(raw string) string { + s := strings.ToLower(strings.TrimSpace(raw)) + if s == "" { + return "" + } + + // Replace punctuation/separators with spaces. + replacer := strings.NewReplacer( + ".", " ", + ",", " ", + ";", " ", + ":", " ", + "(", " ", + ")", " ", + "[", " ", + "]", " ", + "{", " ", + "}", " ", + "/", " ", + "\\", " ", + "-", " ", + "_", " ", + "&", " and ", + "'", "", + "\"", "", + ) + s = replacer.Replace(s) + + // Collapse repeated whitespace. + s = strings.Join(strings.Fields(s), " ") + if s == "" { + return "" + } + + // Remove common legal/company suffixes and generic noise words. + stop := map[string]struct{}{ + "inc": {}, "incorporated": {}, "llc": {}, "l l c": {}, "ltd": {}, "limited": {}, + "corp": {}, "corporation": {}, "co": {}, "company": {}, "gmbh": {}, "ag": {}, + "sa": {}, "s a": {}, "sas": {}, "plc": {}, "pte": {}, "bv": {}, "nv": {}, + "the": {}, "group": {}, "holdings": {}, "technologies": {}, "technology": {}, + "services": {}, "service": {}, + } + + tokens := strings.Fields(s) + out := make([]string, 0, len(tokens)) + for _, t := range tokens { + if _, drop := stop[t]; drop { + continue + } + out = append(out, t) + } + + return strings.Join(out, " ") +} +func canonicalProviderKey(raw string) string { + + var canonicalProviderAliases = map[string][]string{ + "aws-route53": { + "aws route 53", "route 53", "route53", "amazon route 53", "amazon web services", "aws", + }, + "azure-dns": { + "azure", "azure dns", "microsoft", "microsoft azure", + }, + "cloudflare": { + "cloudflare", "cloudflare inc", + }, + "digitalocean": { + "digital ocean", "digitalocean", + }, + "dnsmadeeasy": { + "dns made easy", "dnsmadeeasy", + }, + "dnsimple": { + "dnsimple", + }, + "domaincom": { + "domain com", "domain.com", "domain", + }, + "dreamhost": { + "dreamhost", + }, + "easydns": { + "easydns", "easy dns", + }, + "gandi": { + "gandi", "gandi sas", "gandi.net", + }, + "google-cloud-dns": { + "google cloud", "google cloud dns", "google domains", "googledomains", "google", + }, + "hurricane-electric": { + "hurricane electric", "he net", "he", + }, + "linode": { + "linode", "akamai linode", "akamai", + }, + "namecom": { + "name.com", "name com", "namecom", + }, + "namecheap": { + "namecheap", + }, + "network-solutions": { + "network solutions", "web.com", "webcom", + }, + "ns1": { + "ns1", "nsone", "nsone.net", + }, + "reg-ru": { + "reg ru", "reg.ru", + }, + "tierranet": { + "tierranet", "tierra net", "domaindiscover", + }, + "ultradns": { + "ultradns", "neustar ultradns", "neustar", + }, + "yahoo-smb": { + "yahoo small business", "yahoo", "yns", + }, + } + + s := normalizeOrgName(raw) // lowercase, trim, collapse spaces, strip punctuation/legal suffixes + if s == "" { + return "" + } + for key, aliases := range canonicalProviderAliases { + for _, a := range aliases { + if s == a || strings.Contains(s, a) { + return key + } + } + } + return "" +} diff --git a/scanners/subdomain-takeover/internal/fingerprints/loader_test.go b/scanners/subdomain-takeover/internal/fingerprints/loader_test.go index 68dff54530..f28d2a0d1c 100644 --- a/scanners/subdomain-takeover/internal/fingerprints/loader_test.go +++ b/scanners/subdomain-takeover/internal/fingerprints/loader_test.go @@ -40,7 +40,55 @@ func TestValidateFingerprintHelpers(t *testing.T) { if err := validateCNAMEFingerprints([]CNAMEProviderFingerprint{{Name: "", Cname: []string{"x"}, Fingerprint: "fp"}}); err == nil { t.Fatal("expected error for missing cname fingerprint name") } + if err := validateCNAMEFingerprints([]CNAMEProviderFingerprint{{Name: "ok", Cname: nil, Fingerprint: "fp"}}); err == nil { + t.Fatal("expected error for missing cname patterns") + } + if err := validateCNAMEFingerprints([]CNAMEProviderFingerprint{{Name: "ok", Cname: []string{""}, Fingerprint: "fp"}}); err == nil { + t.Fatal("expected error for empty cname pattern") + } + if err := validateCNAMEFingerprints([]CNAMEProviderFingerprint{{Name: "ok", Cname: []string{"example.net"}, Fingerprint: ""}}); err == nil { + t.Fatal("expected error for missing fingerprint") + } + + validCNAME := []CNAMEProviderFingerprint{{ + Name: "ok", + Cname: []string{"example.net"}, + Fingerprint: "service unavailable", + }} + if err := validateCNAMEFingerprints(validCNAME); err != nil { + t.Fatalf("expected valid cname fingerprints, got error: %v", err) + } + if validCNAME[0].Mode != FingerprintModeLiteral { + t.Fatalf("expected normalized literal mode, got %q", validCNAME[0].Mode) + } + + if err := validateNSFingerprints([]NSProviderFingerprint{{Name: "", HostPatterns: []string{"*.example.net"}}}); err == nil { + t.Fatal("expected error for missing ns name") + } + if err := validateNSFingerprints([]NSProviderFingerprint{{Name: "ok", HostPatterns: nil}}); err == nil { + t.Fatal("expected error for missing ns host patterns") + } if err := validateNSFingerprints([]NSProviderFingerprint{{Name: "ok", HostPatterns: []string{""}}}); err == nil { t.Fatal("expected error for empty ns host pattern") } + if err := validateNSFingerprints([]NSProviderFingerprint{{Name: "ok", HostPatterns: []string{"*.example.net"}}}); err != nil { + t.Fatalf("expected valid ns fingerprints, got error: %v", err) + } +} + +func TestLoadJSON(t *testing.T) { + t.Run("returns read error for missing file", func(t *testing.T) { + var out []CNAMEProviderFingerprint + err := loadJSON("data/missing.json", &out) + if err == nil { + t.Fatal("expected read error") + } + }) + + t.Run("returns decode error for invalid target", func(t *testing.T) { + err := loadJSON("data/cname_fingerprints.json", nil) + if err == nil { + t.Fatal("expected decode error") + } + }) } diff --git a/scanners/subdomain-takeover/internal/messaging/health.go b/scanners/subdomain-takeover/internal/messaging/health.go index 593ca1faff..551e72d825 100644 --- a/scanners/subdomain-takeover/internal/messaging/health.go +++ b/scanners/subdomain-takeover/internal/messaging/health.go @@ -7,10 +7,21 @@ import ( "github.com/nats-io/nats.go" ) +type natsConnectionState interface { + IsClosed() bool + IsConnected() bool + Status() nats.Status +} + func CheckConnection(nc *nats.Conn) error { if nc == nil { return errors.New("nats connection is nil") } + + return checkConnectionState(nc) +} + +func checkConnectionState(nc natsConnectionState) error { if nc.IsClosed() { return errors.New("nats connection is closed") } diff --git a/scanners/subdomain-takeover/internal/messaging/health_test.go b/scanners/subdomain-takeover/internal/messaging/health_test.go index de46741d53..06829d6757 100644 --- a/scanners/subdomain-takeover/internal/messaging/health_test.go +++ b/scanners/subdomain-takeover/internal/messaging/health_test.go @@ -1,6 +1,22 @@ package messaging -import "testing" +import ( + "testing" + + "github.com/nats-io/nats.go" +) + +type fakeConnState struct { + closed bool + connected bool + status nats.Status +} + +func (f fakeConnState) IsClosed() bool { return f.closed } +func (f fakeConnState) IsConnected() bool { return f.connected } +func (f fakeConnState) Status() nats.Status { + return f.status +} func TestCheckConnection(t *testing.T) { err := CheckConnection(nil) @@ -11,3 +27,26 @@ func TestCheckConnection(t *testing.T) { t.Fatalf("unexpected error: %v", err) } } + +func TestCheckConnectionState(t *testing.T) { + t.Run("closed connection returns closed error", func(t *testing.T) { + err := checkConnectionState(fakeConnState{closed: true}) + if err == nil || err.Error() != "nats connection is closed" { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("not connected returns status error", func(t *testing.T) { + err := checkConnectionState(fakeConnState{connected: false, status: nats.DISCONNECTED}) + if err == nil || err.Error() != "nats not connected (status=DISCONNECTED)" { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("connected state returns nil", func(t *testing.T) { + err := checkConnectionState(fakeConnState{connected: true, status: nats.CONNECTED}) + if err != nil { + t.Fatalf("expected nil error, got: %v", err) + } + }) +} diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go index 7ba6e866be..c903f05e00 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -9,6 +9,8 @@ import ( "github.com/rs/zerolog" ) +var marshalFinding = json.Marshal + type Publisher struct { logger zerolog.Logger js publishClient @@ -24,7 +26,7 @@ func NewPublisher(logger zerolog.Logger, js publishClient, subject string) *Publ } func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { - payload, err := json.Marshal(finding) + payload, err := marshalFinding(finding) if err != nil { p.logger.Error(). Err(err). diff --git a/scanners/subdomain-takeover/internal/messaging/publisher_test.go b/scanners/subdomain-takeover/internal/messaging/publisher_test.go index 38de994a00..8595f65017 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher_test.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher_test.go @@ -44,6 +44,9 @@ func TestNewPublisher(t *testing.T) { } func TestPublisherPublish(t *testing.T) { + origMarshal := marshalFinding + t.Cleanup(func() { marshalFinding = origMarshal }) + finding := model.Finding{ Domain: "a.example.ca", DomainKey: "123", @@ -90,4 +93,21 @@ func TestPublisherPublish(t *testing.T) { t.Fatalf("expected one publish call, got %d", client.called) } }) + + t.Run("returns marshal error and does not publish", func(t *testing.T) { + marshalFinding = func(v any) ([]byte, error) { + return nil, errors.New("marshal failed") + } + + client := &fakePublishClient{} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.upsert"} + + err := p.Publish(context.Background(), finding) + if err == nil { + t.Fatal("expected marshal error") + } + if client.called != 0 { + t.Fatalf("expected no publish call, got %d", client.called) + } + }) } From dce38a050928fb69d0376ea8835fbbadbd6dd3de Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 24 Jul 2026 17:51:25 -0300 Subject: [PATCH 43/65] fix formatting --- .../internal/detect/ns_rules_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go index 497dc4160f..61b41f7e47 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go +++ b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go @@ -39,8 +39,8 @@ func TestMatchNSProviderRules(t *testing.T) { t.Run("matches vulnerable full lame and emits correct reason", func(t *testing.T) { evidence := NSEvidence{ - Domain: "a.example.ca", - NSHosts: []string{"ns1.risky-dns.net"}, + Domain: "a.example.ca", + NSHosts: []string{"ns1.risky-dns.net"}, Registrar: namecheapRegistrar, NSDelegations: model.NsDelegations{Delegation: model.Delegation{ LameType: "full", @@ -63,8 +63,8 @@ func TestMatchNSProviderRules(t *testing.T) { t.Run("ranks vulnerable higher than unknown", func(t *testing.T) { evidence := NSEvidence{ - Domain: "a.example.ca", - NSHosts: []string{"ns1.unknown-dns.net", "ns1.risky-dns.net"}, + Domain: "a.example.ca", + NSHosts: []string{"ns1.unknown-dns.net", "ns1.risky-dns.net"}, Registrar: namecheapRegistrar, NSDelegations: model.NsDelegations{Delegation: model.Delegation{ LameType: "partial", @@ -84,8 +84,8 @@ func TestMatchNSProviderRules(t *testing.T) { t.Run("non-vulnerable full lame is classified but not emittable", func(t *testing.T) { evidence := NSEvidence{ - Domain: "a.example.ca", - NSHosts: []string{"ns1.unknown-dns.net"}, + Domain: "a.example.ca", + NSHosts: []string{"ns1.unknown-dns.net"}, Registrar: namecheapRegistrar, NSDelegations: model.NsDelegations{Delegation: model.Delegation{ LameType: "full", From b0ea22978e4266178f73343982ac4607875c08ab Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Wed, 29 Jul 2026 15:25:23 -0300 Subject: [PATCH 44/65] change subject out from upsert to subdomain_takeover --- scanners/subdomain-takeover/README.md | 8 +++++--- .../subdomain-takeover/internal/config/config.go | 2 +- .../internal/messaging/publisher_test.go | 12 ++++++------ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md index 0345e1ec2c..0677e91e3c 100644 --- a/scanners/subdomain-takeover/README.md +++ b/scanners/subdomain-takeover/README.md @@ -7,7 +7,7 @@ Go microservice that consumes DNS scan results and emits normalized subdomain ta 1. Consumes from `scans.dns_scanner_results` (JetStream stream: `SCANS`) 2. Evaluates passive CNAME and NS takeover risk using provider fingerprints 3. Assigns confidence (`suspected`, `probable`, `confirmed`) -4. Publishes findings to `scans.findings.upsert` +4. Publishes findings to `scans.findings.subdomain_takeover` ## Current detection scope @@ -33,6 +33,7 @@ This service uses passive checks only. It does not attempt account takeover, resource claiming, or any active validation. As a result, confidence is conservative and evidence-driven: + - `suspected`: weak or incomplete takeover indicators - `probable`: strong passive indicators with known exploitable conditions - `confirmed`: reserved for deterministic passive signatures only @@ -113,10 +114,11 @@ make ci # lint + test + build ``` Environment variables: + - `NATS_URL` (default: `nats://localhost:4222`) - `NATS_STREAM` (default: `SCANS`) - `SUBJECT_IN` (default: `scans.dns_scanner_results`) -- `SUBJECT_OUT` (default: `scans.findings.upsert`) +- `SUBJECT_OUT` (default: `scans.findings.subdomain_takeover`) - `DURABLE_NAME` (default: `subdomain_takeover`) - `WORKER_COUNT` (default: `5`) - `LOG_LEVEL` (default: `info`) @@ -140,7 +142,7 @@ nats pub scans.dns_scanner_results '{"domain_key":"12345","results":{"domain":"e Watch findings: ```bash -nats sub "scans.findings.upsert" +nats sub "scans.findings.subdomain_takeover" ``` ## Next steps / nice-to-haves diff --git a/scanners/subdomain-takeover/internal/config/config.go b/scanners/subdomain-takeover/internal/config/config.go index 1b18e68117..053993d380 100644 --- a/scanners/subdomain-takeover/internal/config/config.go +++ b/scanners/subdomain-takeover/internal/config/config.go @@ -9,7 +9,7 @@ type Config struct { NATSURL string `envconfig:"NATS_URL" default:"nats://localhost:4222"` NATSStream string `envconfig:"NATS_STREAM" default:"SCANS"` SubjectIn string `envconfig:"SUBJECT_IN" default:"scans.dns_scanner_results"` - SubjectOut string `envconfig:"SUBJECT_OUT" default:"scans.findings.upsert"` + SubjectOut string `envconfig:"SUBJECT_OUT" default:"scans.findings.subdomain_takeover"` DurableName string `envconfig:"DURABLE_NAME" default:"subdomain_takeover"` WorkerCount int `envconfig:"WORKER_COUNT" default:"5"` LogLevel zerolog.Level `envconfig:"LOG_LEVEL" default:"info"` diff --git a/scanners/subdomain-takeover/internal/messaging/publisher_test.go b/scanners/subdomain-takeover/internal/messaging/publisher_test.go index 8595f65017..5e7f83b835 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher_test.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher_test.go @@ -34,11 +34,11 @@ func (f *fakePublishClient) Publish(_ context.Context, subj string, data []byte, func TestNewPublisher(t *testing.T) { client := &fakePublishClient{} - p := NewPublisher(zerolog.Nop(), client, "scans.findings.upsert") + p := NewPublisher(zerolog.Nop(), client, "scans.findings.subdomain_takeover") if p == nil { t.Fatal("expected publisher instance") } - if p.subject != "scans.findings.upsert" { + if p.subject != "scans.findings.subdomain_takeover" { t.Fatalf("unexpected subject: %q", p.subject) } } @@ -59,7 +59,7 @@ func TestPublisherPublish(t *testing.T) { t.Run("publishes serialized finding", func(t *testing.T) { client := &fakePublishClient{} - p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.upsert"} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain_takeover"} err := p.Publish(context.Background(), finding) if err != nil { @@ -68,7 +68,7 @@ func TestPublisherPublish(t *testing.T) { if client.called != 1 { t.Fatalf("expected one publish call, got %d", client.called) } - if client.lastSubj != "scans.findings.upsert" { + if client.lastSubj != "scans.findings.subdomain_takeover" { t.Fatalf("unexpected subject: %q", client.lastSubj) } @@ -83,7 +83,7 @@ func TestPublisherPublish(t *testing.T) { t.Run("returns publish client error", func(t *testing.T) { client := &fakePublishClient{err: errors.New("publish failed")} - p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.upsert"} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain_takeover"} err := p.Publish(context.Background(), finding) if err == nil { @@ -100,7 +100,7 @@ func TestPublisherPublish(t *testing.T) { } client := &fakePublishClient{} - p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.upsert"} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain_takeover"} err := p.Publish(context.Background(), finding) if err == nil { From 433c1d7a8b45041a0898a512ef156ab0a9912313 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 4 Aug 2026 11:28:06 -0300 Subject: [PATCH 45/65] fix: gate RDAP scanning behind feature flag and reduce RDAP scan timeout --- scanners/dns-scanner/.env.example | 2 + .../dns-scanner/dns_scanner/dns_scanner.py | 21 +++-- .../dns-scanner/dns_scanner/ns_registrar.py | 30 ++++--- .../dns-scanner/tests/test_dns_scanner.py | 81 +++++++++++++---- .../dns-scanner/tests/test_ns_registrar.py | 88 +++++++++++++++++++ .../tests/test_scan_domain_registrar_flag.py | 66 ++++++++++++++ 6 files changed, 254 insertions(+), 34 deletions(-) create mode 100644 scanners/dns-scanner/tests/test_ns_registrar.py create mode 100644 scanners/dns-scanner/tests/test_scan_domain_registrar_flag.py diff --git a/scanners/dns-scanner/.env.example b/scanners/dns-scanner/.env.example index 6d3cf0ff2c..9146d956e5 100644 --- a/scanners/dns-scanner/.env.example +++ b/scanners/dns-scanner/.env.example @@ -1,2 +1,4 @@ NATS_SERVERS= SCAN_TIMEOUT= +ENABLE_RDAP_LOOKUP=false +RDAP_TIMEOUT_SEC=3 diff --git a/scanners/dns-scanner/dns_scanner/dns_scanner.py b/scanners/dns-scanner/dns_scanner/dns_scanner.py index 3a2c6fb5a3..b9c53af991 100644 --- a/scanners/dns-scanner/dns_scanner/dns_scanner.py +++ b/scanners/dns-scanner/dns_scanner/dns_scanner.py @@ -12,13 +12,18 @@ from dns.exception import Timeout from dns_scanner.email_scanners import DKIMScanner, DMARCScanner -from dns_scanner.ns_registrar import check_ns_delegations, get_registrar_context +from dns_scanner.ns_registrar import ( + check_ns_delegations, + get_registrar_context, + build_registrar_context, +) logger = logging.getLogger(__name__) TIMEOUT = int(os.getenv("SCAN_TIMEOUT", "20")) DNSSEC_NAMESERVER_IP = os.getenv("DNSSEC_NAMESERVER_IP") DNSSEC_NAMESERVER_HOSTNAME = os.getenv("DNSSEC_NAMESERVER_HOSTNAME") +ENABLE_RDAP_LOOKUP = os.getenv("ENABLE_RDAP_LOOKUP", "false").lower() == "true" @dataclass @@ -332,10 +337,16 @@ def scan_domain(domain, dkim_selectors=None): ) registrar_domain = scan_result.base_domain or zone_apex or domain - scan_result.registrar_context = get_registrar_context( - base_domain=registrar_domain, - ns_hosts=scan_result.ns_delegations.get("ns_hosts", []), - ) + if ENABLE_RDAP_LOOKUP: + scan_result.registrar_context = get_registrar_context( + base_domain=registrar_domain, + ns_hosts=scan_result.ns_delegations.get("ns_hosts", []), + ) + else: + scan_result.registrar_context = build_registrar_context( + base_domain=registrar_domain, + error="rdap_lookup_disabled", + ) # If no MX records are found (with warnings), but there are CNAME records, check the CNAME target for MX records if ( diff --git a/scanners/dns-scanner/dns_scanner/ns_registrar.py b/scanners/dns-scanner/dns_scanner/ns_registrar.py index 4e60d31dc4..7178a2774f 100644 --- a/scanners/dns-scanner/dns_scanner/ns_registrar.py +++ b/scanners/dns-scanner/dns_scanner/ns_registrar.py @@ -7,6 +7,20 @@ from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers TIMEOUT = int(os.getenv("SCAN_TIMEOUT", "20")) +RDAP_TIMEOUT_SEC = float(os.getenv("RDAP_TIMEOUT_SEC", "3")) + + +def build_registrar_context(base_domain, error=None): + return { + "base_domain": base_domain, + "lookup_success": False, + "rdap_url": None, + "registrar_name": None, + "registrar_id": None, + "rdap_nameservers": [], + "delegation_matches_rdap": None, + "error": error, + } def probe_nameserver( @@ -52,8 +66,9 @@ def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_s ns_hosts = ns_records.get("hostnames", []) if len(ns_hosts) == 0: + ns_lookup_name = zone_apex or domain try: - ns_res = resolver.resolve(domain, dns.rdatatype.NS) + ns_res = resolver.resolve(ns_lookup_name, dns.rdatatype.NS) ns_hosts = [host.to_text() for host in ns_res] except (NoAnswer, NXDOMAIN, NoNameservers, Timeout): ns_hosts = [] @@ -126,16 +141,7 @@ def check_ns_delegations(domain, zone_apex, ns_records, resolver=None, timeout_s def get_registrar_context(base_domain, ns_hosts=None): - context = { - "base_domain": base_domain, - "lookup_success": False, - "rdap_url": None, - "registrar_name": None, - "registrar_id": None, - "rdap_nameservers": [], - "delegation_matches_rdap": None, - "error": None, - } + context = build_registrar_context(base_domain) if not base_domain: context["error"] = "missing_base_domain" @@ -145,7 +151,7 @@ def get_registrar_context(base_domain, ns_hosts=None): context["rdap_url"] = rdap_url try: - response = requests.get(rdap_url, timeout=TIMEOUT) + response = requests.get(rdap_url, timeout=RDAP_TIMEOUT_SEC) response.raise_for_status() payload = response.json() except Exception as e: diff --git a/scanners/dns-scanner/tests/test_dns_scanner.py b/scanners/dns-scanner/tests/test_dns_scanner.py index 0b7429ebf3..921d4a0db8 100644 --- a/scanners/dns-scanner/tests/test_dns_scanner.py +++ b/scanners/dns-scanner/tests/test_dns_scanner.py @@ -1,24 +1,71 @@ -import pytest -from pretend import stub -from starlette.testclient import TestClient -from server import Server +from dns_scanner import dns_scanner as scanner_mod -def test_scan(): - client_stub = stub(post=lambda url, json: None) +class FakeResolver: + timeout = None + lifetime = None - test_app = Server(server_client=client_stub) + def resolve(self, *args, **kwargs): + raise scanner_mod.NoAnswer - test_client = TestClient(test_app) - test_payload = { - "user_key": 1, - "domain": "cyber.gc.ca", - "domain_key": "domains/1", - "selectors": ["selector1", "selector2"], - "shared_id": 1234 - } +class FakeDMARCScanner: + def __init__(self, domain): + self.domain = domain - res = test_client.post("/", json=test_payload) + def run(self): + return { + "base_domain": "example.com", + "mx": {"hosts": [], "warnings": []}, + "spf": {}, + "dmarc": {}, + "ns": {"hostnames": ["ns1.example.com."], "errors": []}, + } - assert "Scan completed" == res.text + +class FakeDKIMScanner: + def __init__(self, domain, selectors): + self.domain = domain + self.selectors = selectors + + def run(self): + return {"selectors": self.selectors} + + +def test_scan_domain_returns_expected_shape(monkeypatch): + monkeypatch.setattr(scanner_mod, "ENABLE_RDAP_LOOKUP", False) + monkeypatch.setattr( + scanner_mod, + "get_dns_return_type", + lambda domain, query_type: "NOERROR", + ) + monkeypatch.setattr(scanner_mod, "find_zone_apex", lambda domain: "example.com") + monkeypatch.setattr(scanner_mod, "DMARCScanner", FakeDMARCScanner) + monkeypatch.setattr(scanner_mod, "DKIMScanner", FakeDKIMScanner) + monkeypatch.setattr(scanner_mod.dns.resolver, "Resolver", lambda: FakeResolver()) + monkeypatch.setattr( + scanner_mod, + "get_wildcard_status", + lambda domain, resolver, a_records: { + "wildcard_entry": False, + "wildcard_sibling": False, + }, + ) + monkeypatch.setattr( + scanner_mod, + "check_ns_delegations", + lambda domain, zone_apex, ns_records: { + "ns_hosts": ns_records.get("hostnames", []), + "ns_checks": [], + "ns_delegation": {"lame_type": "none"}, + }, + ) + + result = scanner_mod.scan_domain("mail.example.com", dkim_selectors=["selector1"]) + + assert result["record_exists"] is True + assert result["rcode"] == "NOERROR" + assert result["zone_apex"] == "example.com" + assert result["base_domain"] == "example.com" + assert result["dkim"] == {"selectors": ["selector1"]} + assert result["registrar_context"]["error"] == "rdap_lookup_disabled" diff --git a/scanners/dns-scanner/tests/test_ns_registrar.py b/scanners/dns-scanner/tests/test_ns_registrar.py new file mode 100644 index 0000000000..c04da55a5f --- /dev/null +++ b/scanners/dns-scanner/tests/test_ns_registrar.py @@ -0,0 +1,88 @@ +import dns + +from dns_scanner import ns_registrar + + +class FakeHost: + def __init__(self, value): + self.value = value + + def to_text(self): + return self.value + + +class FakeResponse: + def __init__(self, rcode, authoritative): + self._rcode = rcode + self.flags = dns.flags.AA if authoritative else 0 + + def rcode(self): + return self._rcode + + +class RecordingResolver: + def __init__(self): + self.calls = [] + + def resolve(self, name, query_type): + self.calls.append((name, query_type)) + if query_type == dns.rdatatype.NS: + return [FakeHost("ns1.example.com.")] + raise AssertionError(f"Unexpected resolve call: {(name, query_type)}") + + +def test_check_ns_delegations_fallback_queries_zone_apex(monkeypatch): + resolver = RecordingResolver() + + monkeypatch.setattr(ns_registrar, "get_ns_ip", lambda host, resolver: "192.0.2.1") + def fake_probe(where, qname, qtype, recursion_desired, timeout): + assert qname == "example.com" + assert qtype == "SOA" + return FakeResponse(dns.rcode.NOERROR, True) + + monkeypatch.setattr(ns_registrar, "probe_nameserver", fake_probe) + + result = ns_registrar.check_ns_delegations( + domain="mail.example.com", + zone_apex="example.com", + ns_records={"hostnames": [], "errors": []}, + resolver=resolver, + ) + + assert resolver.calls[0] == ("example.com", dns.rdatatype.NS) + assert result["ns_delegation"]["lame_type"] == "none" + assert result["ns_delegation"]["authoritative_ok"] == 1 + + +def test_build_registrar_context_defaults(): + context = ns_registrar.build_registrar_context("example.com") + + assert context["base_domain"] == "example.com" + assert context["lookup_success"] is False + assert context["error"] is None + assert context["rdap_nameservers"] == [] + + +def test_get_registrar_context_uses_rdap_timeout(monkeypatch): + captured = {} + + class FakeRdapResponse: + def raise_for_status(self): + return None + + def json(self): + return {"nameservers": [], "entities": []} + + def fake_get(url, timeout): + captured["url"] = url + captured["timeout"] = timeout + return FakeRdapResponse() + + monkeypatch.setattr(ns_registrar, "RDAP_TIMEOUT_SEC", 2.5) + monkeypatch.setattr(ns_registrar.requests, "get", fake_get) + + context = ns_registrar.get_registrar_context("example.com", ns_hosts=[]) + + assert captured["url"] == "https://rdap.org/domain/example.com" + assert captured["timeout"] == 2.5 + assert context["lookup_success"] is True diff --git a/scanners/dns-scanner/tests/test_scan_domain_registrar_flag.py b/scanners/dns-scanner/tests/test_scan_domain_registrar_flag.py new file mode 100644 index 0000000000..f1aa2cbfca --- /dev/null +++ b/scanners/dns-scanner/tests/test_scan_domain_registrar_flag.py @@ -0,0 +1,66 @@ +from dns_scanner import dns_scanner as scanner_mod + + +class FakeResolver: + timeout = None + lifetime = None + + def resolve(self, *args, **kwargs): + raise scanner_mod.NoAnswer + + +class FakeDMARCScanner: + def __init__(self, domain): + self.domain = domain + + def run(self): + return { + "base_domain": "example.com", + "mx": {"hosts": [], "warnings": []}, + "spf": {}, + "dmarc": {}, + "ns": {"hostnames": [], "errors": []}, + } + + +class FakeDKIMScanner: + def __init__(self, domain, selectors): + self.domain = domain + self.selectors = selectors + + def run(self): + return {} + + +def test_scan_domain_skips_rdap_lookup_when_disabled(monkeypatch): + monkeypatch.setattr(scanner_mod, "ENABLE_RDAP_LOOKUP", False) + monkeypatch.setattr(scanner_mod, "get_dns_return_type", lambda domain, query_type: "NOERROR") + monkeypatch.setattr(scanner_mod, "find_zone_apex", lambda domain: "example.com") + monkeypatch.setattr( + scanner_mod, + "check_ns_delegations", + lambda domain, zone_apex, ns_records: {"ns_hosts": [], "ns_checks": [], "ns_delegation": {}}, + ) + monkeypatch.setattr( + scanner_mod, + "get_registrar_context", + lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("should not be called")), + ) + monkeypatch.setattr(scanner_mod, "DMARCScanner", FakeDMARCScanner) + monkeypatch.setattr(scanner_mod, "DKIMScanner", FakeDKIMScanner) + monkeypatch.setattr(scanner_mod.dns.resolver, "Resolver", lambda: FakeResolver()) + monkeypatch.setattr( + scanner_mod, + "get_wildcard_status", + lambda domain, resolver, a_records: { + "wildcard_entry": False, + "wildcard_sibling": False, + }, + ) + + result = scanner_mod.scan_domain("mail.example.com", dkim_selectors=[]) + + assert result["record_exists"] is True + assert result["registrar_context"]["base_domain"] == "example.com" + assert result["registrar_context"]["lookup_success"] is False + assert result["registrar_context"]["error"] == "rdap_lookup_disabled" From d76162847d8e6ffb754d95fdefb1daae56926bda Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 4 Aug 2026 14:19:43 -0300 Subject: [PATCH 46/65] subdomain-takeover: add skeleton, runtime wiring, and ops config --- scanners/subdomain-takeover/.env.example | 7 ++ scanners/subdomain-takeover/Dockerfile | 22 +++++ scanners/subdomain-takeover/Makefile | 47 +++++++++ scanners/subdomain-takeover/cloudbuild.yaml | 43 ++++++++ .../subdomain-takeover/cmd/service/main.go | 87 +++++++++++++++++ scanners/subdomain-takeover/go.mod | 19 ++++ scanners/subdomain-takeover/go.sum | 21 ++++ .../subdomain-takeover/internal/app/runner.go | 97 +++++++++++++++++++ .../subdomain-takeover/internal/app/worker.go | 95 ++++++++++++++++++ .../internal/bootstrap/logger.go | 13 +++ .../internal/bootstrap/nats.go | 65 +++++++++++++ .../internal/config/config.go | 25 +++++ .../internal/messaging/health.go | 32 ++++++ .../internal/messaging/publisher.go | 64 ++++++++++++ .../internal/model/finding.go | 19 ++++ .../internal/model/input.go | 55 +++++++++++ 16 files changed, 711 insertions(+) create mode 100644 scanners/subdomain-takeover/.env.example create mode 100644 scanners/subdomain-takeover/Dockerfile create mode 100644 scanners/subdomain-takeover/Makefile create mode 100644 scanners/subdomain-takeover/cloudbuild.yaml create mode 100644 scanners/subdomain-takeover/cmd/service/main.go create mode 100644 scanners/subdomain-takeover/go.mod create mode 100644 scanners/subdomain-takeover/go.sum create mode 100644 scanners/subdomain-takeover/internal/app/runner.go create mode 100644 scanners/subdomain-takeover/internal/app/worker.go create mode 100644 scanners/subdomain-takeover/internal/bootstrap/logger.go create mode 100644 scanners/subdomain-takeover/internal/bootstrap/nats.go create mode 100644 scanners/subdomain-takeover/internal/config/config.go create mode 100644 scanners/subdomain-takeover/internal/messaging/health.go create mode 100644 scanners/subdomain-takeover/internal/messaging/publisher.go create mode 100644 scanners/subdomain-takeover/internal/model/finding.go create mode 100644 scanners/subdomain-takeover/internal/model/input.go diff --git a/scanners/subdomain-takeover/.env.example b/scanners/subdomain-takeover/.env.example new file mode 100644 index 0000000000..31d2f06813 --- /dev/null +++ b/scanners/subdomain-takeover/.env.example @@ -0,0 +1,7 @@ +NATS_URL= +NATS_STREAM= +SUBJECT_IN= +SUBJECT_OUT= +DURABLE_NAME= +WORKER_COUNT= +LOG_LEVEL= diff --git a/scanners/subdomain-takeover/Dockerfile b/scanners/subdomain-takeover/Dockerfile new file mode 100644 index 0000000000..6c7f3eafaa --- /dev/null +++ b/scanners/subdomain-takeover/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.25-alpine AS builder + +WORKDIR /src +RUN apk add --no-cache git ca-certificates + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/subdomain-takeover ./cmd/service + +# Runtime stage +FROM alpine:3.22 + +RUN addgroup -S scanner && adduser -S scanner -G scanner +WORKDIR /app +RUN apk add --no-cache ca-certificates tzdata + +COPY --from=builder /out/subdomain-takeover /app/subdomain-takeover + +USER scanner +ENTRYPOINT ["/app/subdomain-takeover"] diff --git a/scanners/subdomain-takeover/Makefile b/scanners/subdomain-takeover/Makefile new file mode 100644 index 0000000000..72345ab0b2 --- /dev/null +++ b/scanners/subdomain-takeover/Makefile @@ -0,0 +1,47 @@ +.PHONY: help run test test-race build fmt fmt-check vet lint tidy ci + +GO ?= go +SERVICE_BIN ?= subdomain-takeover +BUILD_DIR ?= bin + +help: + @printf "Targets:\n" + @printf " make run - Run the service\n" + @printf " make test - Run all tests\n" + @printf " make test-race - Run tests with race detector\n" + @printf " make build - Build service binary\n" + @printf " make fmt - Format Go files\n" + @printf " make fmt-check - Check formatting (no changes)\n" + @printf " make vet - Run go vet\n" + @printf " make lint - Run fmt-check + vet\n" + @printf " make tidy - Tidy modules\n" + @printf " make ci - Lint, test, build\n" + +run: + $(GO) run ./cmd/service + +test: + $(GO) test ./... + +test-race: + $(GO) test -race ./... + +build: + mkdir -p $(BUILD_DIR) + CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(SERVICE_BIN) ./cmd/service + +fmt: + $(GO) fmt ./... + +fmt-check: + @test -z "$$($(GO)fmt -l .)" || (printf "Unformatted files found. Run 'make fmt'.\n" && exit 1) + +vet: + $(GO) vet ./... + +lint: fmt-check vet + +tidy: + $(GO) mod tidy + +ci: lint test build diff --git a/scanners/subdomain-takeover/cloudbuild.yaml b/scanners/subdomain-takeover/cloudbuild.yaml new file mode 100644 index 0000000000..64bd3d7902 --- /dev/null +++ b/scanners/subdomain-takeover/cloudbuild.yaml @@ -0,0 +1,43 @@ +steps: + - name: "golang:1.25" + id: ci-checks + dir: scanners/subdomain-takeover + entrypoint: "bash" + args: + - "-c" + - | + make ci + + - name: "gcr.io/cloud-builders/docker" + id: generate-image-name + entrypoint: "bash" + dir: scanners/subdomain-takeover + args: + - "-c" + - | + echo "northamerica-northeast1-docker.pkg.dev/track-compliance/tracker/subdomain-takeover:$(echo $BRANCH_NAME | sed 's/[^a-zA-Z0-9]/-/g')-$SHORT_SHA-$(date +%s)" > /workspace/imagename + + - name: "gcr.io/cloud-builders/docker" + id: build-scanner + entrypoint: "bash" + dir: scanners/subdomain-takeover + args: + - "-c" + - | + image=$(cat /workspace/imagename) + docker build -t $image . + + - name: "gcr.io/cloud-builders/docker" + id: push-scanner-if-master + entrypoint: "bash" + dir: scanners/subdomain-takeover + args: + - "-c" + - | + if [[ "$BRANCH_NAME" == "master" ]] + then + image=$(cat /workspace/imagename) + docker push $image + else + exit 0 + fi diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go new file mode 100644 index 0000000000..816ab0cc80 --- /dev/null +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -0,0 +1,87 @@ +package main + +import ( + "context" + "os" + "os/signal" + "syscall" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/app" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/bootstrap" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" + "github.com/rs/zerolog" +) + +func main() { + logger := bootstrap.NewLogger(zerolog.InfoLevel) + + cfg, err := config.Load() + if err != nil { + logger.Fatal().Err(err).Msg("failed to load config") + } + + logger = bootstrap.NewLogger(cfg.LogLevel) + logger.Info(). + Str("nats_url", cfg.NATSURL). + Str("stream", cfg.NATSStream). + Str("subject_in", cfg.SubjectIn). + Str("subject_out", cfg.SubjectOut). + Str("durable", cfg.DurableName). + Int("worker_count", cfg.WorkerCount). + Str("log_level", cfg.LogLevel.String()). + Msg("service configuration loaded") + + if err := fingerprints.Load(logger); err != nil { + logger.Fatal().Err(err).Msg("failed to load fingerprints") + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + defer signal.Stop(sig) + + runtimeDeps, err := bootstrap.NewRuntimeDeps(ctx, cfg, logger) + if err != nil { + logger.Fatal().Err(err).Msg("failed to initialize runtime dependencies") + } + + logger.Info().Msgf("Connected to NATS at %s", runtimeDeps.NC.ConnectedUrl()) + + pub := messaging.NewPublisher(logger, runtimeDeps.JS, cfg.SubjectOut) + matcher := detect.NewHTTPBodyFingerprintMatcher(5 * time.Second) + classifier := detect.NewClassifier(matcher).WithLogger(logger) + worker := app.NewWorker(logger, pub, classifier) + + go func() { + <-sig + logger.Info().Msg("Shutdown requested...") + cancel() + runtimeDeps.Iter.Stop() + }() + + deps := app.RunnerDeps{ + Logger: logger, + WorkerCount: cfg.WorkerCount, + Iter: runtimeDeps.Iter, + Worker: worker, + NC: runtimeDeps.NC, + } + + app.Run(ctx, deps) + + logger.Info().Msgf("Disconnecting from NATS at %s", runtimeDeps.NC.ConnectedUrl()) + if err := runtimeDeps.NC.Flush(); err != nil { + logger.Error().Err(err).Msg("failed to flush nats connection") + } + runtimeDeps.NC.Close() + + if err := ctx.Err(); err != nil && err != context.Canceled { + logger.Error().Err(err).Msg("service exited with context error") + } +} diff --git a/scanners/subdomain-takeover/go.mod b/scanners/subdomain-takeover/go.mod new file mode 100644 index 0000000000..ca7b20f4ac --- /dev/null +++ b/scanners/subdomain-takeover/go.mod @@ -0,0 +1,19 @@ +module github.com/canada-ca/tracker/scanners/subdomain-takeover + +go 1.25.0 + +require ( + github.com/kelseyhightower/envconfig v1.4.0 + github.com/nats-io/nats.go v1.52.0 + github.com/rs/zerolog v1.35.1 +) + +require ( + github.com/klauspost/compress v1.18.5 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.49.0 // indirect + golang.org/x/sys v0.42.0 // indirect +) diff --git a/scanners/subdomain-takeover/go.sum b/scanners/subdomain-takeover/go.sum new file mode 100644 index 0000000000..470146821e --- /dev/null +++ b/scanners/subdomain-takeover/go.sum @@ -0,0 +1,21 @@ +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= +github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= +github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= +github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go new file mode 100644 index 0000000000..924427b855 --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -0,0 +1,97 @@ +package app + +import ( + "context" + "sync" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type RunnerDeps struct { + Logger zerolog.Logger + WorkerCount int + Iter jetstream.MessagesContext + Worker MessageHandler + NC *nats.Conn +} + +type MessageHandler interface { + Handle(ctx context.Context, msg jetstream.Msg) error +} + +var checkConnection = messaging.CheckConnection + +func Run(ctx context.Context, deps RunnerDeps) { + logger := deps.Logger + iter := deps.Iter + nextErrCount := 0 + + if deps.WorkerCount < 1 { + deps.WorkerCount = 1 + } + + logger.Info().Int("worker_count", deps.WorkerCount).Msg("runner started") + + var wg sync.WaitGroup + sem := make(chan struct{}, deps.WorkerCount) + +Loop: + for { + if err := checkConnection(deps.NC); err != nil { + logger.Error().Err(err).Msg("NATS connection unhealthy") + break Loop + } + + select { + case <-ctx.Done(): + break Loop + default: + } + + msg, err := iter.Next() + if err != nil { + if ctx.Err() != nil { + logger.Info().Msg("runner stopping: context canceled") + break Loop + } + nextErrCount++ + if nextErrCount%10 == 0 { + logger.Warn().Err(err).Int("consecutive_next_errors", nextErrCount).Msg("iterator next repeatedly failed") + } else { + logger.Debug().Err(err).Int("consecutive_next_errors", nextErrCount).Msg("next returned, continuing") + } + continue + } + nextErrCount = 0 + + select { + case sem <- struct{}{}: + case <-ctx.Done(): + break Loop + } + + wg.Add(1) + go func(m jetstream.Msg) { + defer wg.Done() + defer func() { <-sem }() + deps.Worker.Handle(ctx, m) + }(msg) + } + + waitDone := make(chan struct{}) + go func() { + wg.Wait() + close(waitDone) + }() + + select { + case <-waitDone: + logger.Info().Msg("all workers drained") + case <-time.After(30 * time.Second): + logger.Warn().Msg("drain timeout") + } +} diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go new file mode 100644 index 0000000000..eca59f91bc --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -0,0 +1,95 @@ +package app + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type FindingPublisher interface { + Publish(ctx context.Context, finding model.Finding) error +} + +type ScanClassifier interface { + Classify(input model.Input) ([]model.Finding, error) +} + +type Worker struct { + logger zerolog.Logger + publisher FindingPublisher + classifier ScanClassifier +} + +func NewWorker(logger zerolog.Logger, publisher FindingPublisher, classifier ScanClassifier) *Worker { + return &Worker{logger: logger, publisher: publisher, classifier: classifier} +} + +func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { + log := w.logger.With(). + Str("component", "worker"). + Str("subject", msg.Subject()). + Int("msg_size", len(msg.Data())). + Logger() + + scan, err := decodeScan(msg.Data()) + if err != nil { + log.Err(err).Msg("decode error") + w.term(msg, log, err) + return err + } + + log = log.With().Str("domain_key", scan.DomainKey).Logger() + + findings, err := w.classifier.Classify(scan) + if err != nil { + log.Err(err).Msg("classify error") + w.nak(msg, log, err) + return err + } + + if len(findings) == 0 { + log.Debug().Msg("classification produced no findings") + } + + for _, finding := range findings { + err = w.publisher.Publish(ctx, finding) + if err != nil { + log.Err(err).Msg("publish error") + w.nak(msg, log, err) + return err + } + } + + if err := msg.Ack(); err != nil { + log.Err(err).Msg("ack error") + return err + } + + log.Debug().Int("findings_count", len(findings)).Msg("message acknowledged") + + return nil +} + +func decodeScan(data []byte) (model.Input, error) { + var scan model.Input + trimmed := bytes.Trim(data, "\n") + err := json.Unmarshal(trimmed, &scan) + return scan, err +} + +func (w *Worker) nak(msg jetstream.Msg, log zerolog.Logger, originalErr error) { + if err := msg.Nak(); err != nil { + log.Error().Err(fmt.Errorf("original=%v nak=%w", originalErr, err)).Msg("failed to nak message") + } +} + +func (w *Worker) term(msg jetstream.Msg, log zerolog.Logger, originalErr error) { + if err := msg.Term(); err != nil { + log.Error().Err(fmt.Errorf("original=%v term=%w", originalErr, err)).Msg("failed to term message") + } +} diff --git a/scanners/subdomain-takeover/internal/bootstrap/logger.go b/scanners/subdomain-takeover/internal/bootstrap/logger.go new file mode 100644 index 0000000000..3721d23e40 --- /dev/null +++ b/scanners/subdomain-takeover/internal/bootstrap/logger.go @@ -0,0 +1,13 @@ +package bootstrap + +import ( + "os" + + "github.com/rs/zerolog" +) + +func NewLogger(level zerolog.Level) zerolog.Logger { + zerolog.SetGlobalLevel(level) + multi := zerolog.MultiLevelWriter(zerolog.ConsoleWriter{Out: os.Stderr}) + return zerolog.New(multi).With().Timestamp().Logger() +} diff --git a/scanners/subdomain-takeover/internal/bootstrap/nats.go b/scanners/subdomain-takeover/internal/bootstrap/nats.go new file mode 100644 index 0000000000..ea2fc49e21 --- /dev/null +++ b/scanners/subdomain-takeover/internal/bootstrap/nats.go @@ -0,0 +1,65 @@ +package bootstrap + +import ( + "context" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type RuntimeDeps struct { + NC *nats.Conn + Iter jetstream.MessagesContext + JS jetstream.JetStream +} + +func NewRuntimeDeps(ctx context.Context, cfg *config.Config, logger zerolog.Logger) (*RuntimeDeps, error) { + logger.Debug().Str("nats_url", cfg.NATSURL).Msg("connecting to nats") + nc, err := nats.Connect(cfg.NATSURL) + if err != nil { + return nil, err + } + + logger.Debug().Msg("creating jetstream client") + js, err := jetstream.New(nc) + if err != nil { + nc.Close() + return nil, err + } + + logger.Debug().Str("stream", cfg.NATSStream).Strs("subjects", []string{cfg.SubjectIn, cfg.SubjectOut}).Msg("creating or updating stream") + stream, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ + Name: cfg.NATSStream, + Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, + }) + if err != nil { + nc.Close() + return nil, err + } + + logger.Debug().Str("durable", cfg.DurableName).Msg("creating or updating consumer") + cons, err := stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: cfg.DurableName, + AckPolicy: jetstream.AckExplicitPolicy, + }) + if err != nil { + nc.Close() + return nil, err + } + + logger.Debug().Msg("creating pull message iterator") + iter, err := cons.Messages(jetstream.PullMaxMessages(1), jetstream.PullExpiry(1*time.Second)) + if err != nil { + nc.Close() + return nil, err + } + + return &RuntimeDeps{ + NC: nc, + Iter: iter, + JS: js, + }, nil +} diff --git a/scanners/subdomain-takeover/internal/config/config.go b/scanners/subdomain-takeover/internal/config/config.go new file mode 100644 index 0000000000..053993d380 --- /dev/null +++ b/scanners/subdomain-takeover/internal/config/config.go @@ -0,0 +1,25 @@ +package config + +import ( + "github.com/kelseyhightower/envconfig" + "github.com/rs/zerolog" +) + +type Config struct { + NATSURL string `envconfig:"NATS_URL" default:"nats://localhost:4222"` + NATSStream string `envconfig:"NATS_STREAM" default:"SCANS"` + SubjectIn string `envconfig:"SUBJECT_IN" default:"scans.dns_scanner_results"` + SubjectOut string `envconfig:"SUBJECT_OUT" default:"scans.findings.subdomain_takeover"` + DurableName string `envconfig:"DURABLE_NAME" default:"subdomain_takeover"` + WorkerCount int `envconfig:"WORKER_COUNT" default:"5"` + LogLevel zerolog.Level `envconfig:"LOG_LEVEL" default:"info"` +} + +func Load() (*Config, error) { + var cfg Config + if err := envconfig.Process("", &cfg); err != nil { + return nil, err + } + + return &cfg, nil +} diff --git a/scanners/subdomain-takeover/internal/messaging/health.go b/scanners/subdomain-takeover/internal/messaging/health.go new file mode 100644 index 0000000000..551e72d825 --- /dev/null +++ b/scanners/subdomain-takeover/internal/messaging/health.go @@ -0,0 +1,32 @@ +package messaging + +import ( + "errors" + "fmt" + + "github.com/nats-io/nats.go" +) + +type natsConnectionState interface { + IsClosed() bool + IsConnected() bool + Status() nats.Status +} + +func CheckConnection(nc *nats.Conn) error { + if nc == nil { + return errors.New("nats connection is nil") + } + + return checkConnectionState(nc) +} + +func checkConnectionState(nc natsConnectionState) error { + if nc.IsClosed() { + return errors.New("nats connection is closed") + } + if !nc.IsConnected() { + return fmt.Errorf("nats not connected (status=%s)", nc.Status().String()) + } + return nil +} diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go new file mode 100644 index 0000000000..c903f05e00 --- /dev/null +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -0,0 +1,64 @@ +package messaging + +import ( + "context" + "encoding/json" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +var marshalFinding = json.Marshal + +type Publisher struct { + logger zerolog.Logger + js publishClient + subject string +} + +type publishClient interface { + Publish(ctx context.Context, subj string, data []byte, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error) +} + +func NewPublisher(logger zerolog.Logger, js publishClient, subject string) *Publisher { + return &Publisher{logger: logger, js: js, subject: subject} +} + +func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { + payload, err := marshalFinding(finding) + if err != nil { + p.logger.Error(). + Err(err). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Str("reason_code", finding.ReasonCode). + Str("confidence", finding.Confidence). + Msg("marshal finding failed") + return err + } + + if _, err := p.js.Publish(ctx, p.subject, payload); err != nil { + p.logger.Error(). + Err(err). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Str("reason_code", finding.ReasonCode). + Str("confidence", finding.Confidence). + Str("subject", p.subject). + Msg("publish failed") + return err + } + + p.logger.Debug(). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Str("reason_code", finding.ReasonCode). + Str("confidence", finding.Confidence). + Str("subject", p.subject). + Msg("finding published") + return nil +} diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go new file mode 100644 index 0000000000..66e98a2f35 --- /dev/null +++ b/scanners/subdomain-takeover/internal/model/finding.go @@ -0,0 +1,19 @@ +package model + +type RecordType string + +const ( + RecordTypeCNAME RecordType = "CNAME" + RecordTypeNS RecordType = "NS" +) + +type Finding struct { + Domain string `json:"domain"` + DomainKey string `json:"domain_key"` + RecordType RecordType `json:"record_type"` + Target string `json:"target"` + Provider string `json:"provider"` + LameType string `json:"lame_type"` + Confidence string `json:"confidence"` + ReasonCode string `json:"reason_code"` +} diff --git a/scanners/subdomain-takeover/internal/model/input.go b/scanners/subdomain-takeover/internal/model/input.go new file mode 100644 index 0000000000..263e74928d --- /dev/null +++ b/scanners/subdomain-takeover/internal/model/input.go @@ -0,0 +1,55 @@ +package model + +type Input struct { + DomainKey string `json:"domain_key"` + Results ScanResults `json:"results"` +} + +type ScanResults struct { + Domain *string `json:"domain"` + NsDelegations *NsDelegations `json:"ns_delegations"` + RegistrarContext *RegistrarContext `json:"registrar_context"` + ResolveChain [][]string `json:"resolve_chain"` + CnameRecord *string `json:"cname_record"` + NsRecords *NsRecords `json:"ns_records"` +} + +type NsRecords struct { + Hostnames []string `json:"hostnames"` + Warnings []string `json:"warnings"` + Error string `json:"error"` +} + +type NsDelegations struct { + Hosts []string `json:"ns_hosts"` + Checks []NsCheck `json:"ns_checks"` + Delegation Delegation `json:"ns_delegation"` + Error string `json:"error"` +} + +type NsCheck struct { + Host string `json:"ns_host"` + Qname string `json:"qname"` + Qtype string `json:"qtype"` + Rcode string `json:"rcode"` + AnsweredAuthoritatively bool `json:"answered_authoritatively"` + Error *string `json:"error"` + Timeout bool `json:"timeout"` +} +type Delegation struct { + TotalHosts int `json:"total_ns"` + OkCount int `json:"authoritative_ok"` + LameCount int `json:"lame_count"` + LameType string `json:"lame_type"` +} + +type RegistrarContext struct { + BaseDomain string `json:"base_domain"` + LookupSuccess bool `json:"lookup_success"` + RDAPURL string `json:"rdap_url"` + RegistrarName string `json:"registrar_name"` + RegistrarID string `json:"registrar_id"` + RDAPNameservers []string `json:"rdap_nameservers"` + DelegationMatchesRDAP *bool `json:"delegation_matches_rdap"` + Error string `json:"error"` +} From acfeda6bf752ef93fb7d4907aee33495b988d659 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 4 Aug 2026 14:19:54 -0300 Subject: [PATCH 47/65] subdomain-takeover: add final detection architecture and rules --- .../internal/detect/body_matcher.go | 67 +++++++++ .../internal/detect/classifier.go | 129 ++++++++++++++++ .../internal/detect/cname_evidence.go | 31 ++++ .../internal/detect/cname_rules.go | 79 ++++++++++ .../internal/detect/confidence.go | 37 +++++ .../internal/detect/fingerprint_source.go | 18 +++ .../internal/detect/logger.go | 9 ++ .../internal/detect/ns_evidence.go | 43 ++++++ .../internal/detect/ns_reasoning.go | 95 ++++++++++++ .../internal/detect/ns_rules.go | 75 ++++++++++ .../internal/detect/reason_codes.go | 17 +++ .../internal/detect/registrar_helpers.go | 139 ++++++++++++++++++ 12 files changed, 739 insertions(+) create mode 100644 scanners/subdomain-takeover/internal/detect/body_matcher.go create mode 100644 scanners/subdomain-takeover/internal/detect/classifier.go create mode 100644 scanners/subdomain-takeover/internal/detect/cname_evidence.go create mode 100644 scanners/subdomain-takeover/internal/detect/cname_rules.go create mode 100644 scanners/subdomain-takeover/internal/detect/confidence.go create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprint_source.go create mode 100644 scanners/subdomain-takeover/internal/detect/logger.go create mode 100644 scanners/subdomain-takeover/internal/detect/ns_evidence.go create mode 100644 scanners/subdomain-takeover/internal/detect/ns_reasoning.go create mode 100644 scanners/subdomain-takeover/internal/detect/ns_rules.go create mode 100644 scanners/subdomain-takeover/internal/detect/reason_codes.go create mode 100644 scanners/subdomain-takeover/internal/detect/registrar_helpers.go diff --git a/scanners/subdomain-takeover/internal/detect/body_matcher.go b/scanners/subdomain-takeover/internal/detect/body_matcher.go new file mode 100644 index 0000000000..b28d4b496e --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/body_matcher.go @@ -0,0 +1,67 @@ +package detect + +import ( + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" +) + +type BodyFingerprintMatcher interface { + Contains(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool +} + +type HTTPBodyFingerprintMatcher struct { + client *http.Client +} + +func NewHTTPBodyFingerprintMatcher(timeout time.Duration) *HTTPBodyFingerprintMatcher { + return &HTTPBodyFingerprintMatcher{ + client: &http.Client{Timeout: timeout}, + } +} + +type NoopBodyFingerprintMatcher struct{} + +func NewNoopBodyFingerprintMatcher() *NoopBodyFingerprintMatcher { + return &NoopBodyFingerprintMatcher{} +} + +func (m *NoopBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { + return false +} + +func (m *HTTPBodyFingerprintMatcher) Contains(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { + if m == nil || m.client == nil { + return false + } + + url := fmt.Sprintf("http://%s", domain) + res, err := m.client.Get(url) + if err != nil { + return false + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return false + } + + bodyText := string(body) + resolvedMode := fingerprints.NormalizeMode(mode, fingerprint) + + if resolvedMode == fingerprints.FingerprintModeRegex { + re, err := regexp.Compile(fingerprint) + if err != nil { + return false + } + return re.MatchString(bodyText) + } + + return strings.Contains(bodyText, fingerprint) +} diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go new file mode 100644 index 0000000000..34ccd35246 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -0,0 +1,129 @@ +package detect + +import ( + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/rs/zerolog" +) + +type Classifier struct { + Matcher BodyFingerprintMatcher + Source FingerprintSource + logger zerolog.Logger +} + +func NewClassifier(matcher BodyFingerprintMatcher) *Classifier { + return &Classifier{Matcher: matcher, Source: GlobalFingerprintSource{}, logger: zerolog.Nop()} +} + +func NewClassifierWithSource(matcher BodyFingerprintMatcher, source FingerprintSource) *Classifier { + if source == nil { + source = GlobalFingerprintSource{} + } + + return &Classifier{Matcher: matcher, Source: source, logger: zerolog.Nop()} +} + +func (c *Classifier) WithLogger(logger zerolog.Logger) *Classifier { + c.logger = logger.With().Str("component", "classifier").Logger() + SetLogger(logger) + return c +} + +func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { + matcher := c.Matcher + if matcher == nil { + matcher = NewNoopBodyFingerprintMatcher() + } + + source := c.Source + if source == nil { + source = GlobalFingerprintSource{} + } + + logger := c.logger + if logger.GetLevel() == zerolog.NoLevel { + logger = zerolog.Nop() + } + + return Classify(input, matcher, source, logger) +} + +func Classify(input model.Input, matcher BodyFingerprintMatcher, source FingerprintSource, logger zerolog.Logger) ([]model.Finding, error) { + findings := []model.Finding{} + + cnameProviderFingerprints := source.CNAME() + nsProviderFingerprints := source.NS() + + cnameEvidence := ExtractCNAMEEvidence(input.Results) + if cnameEvidence != nil { + logger.Debug().Str("domain_key", input.DomainKey).Str("domain", cnameEvidence.Domain).Msg("cname evidence extracted") + cnameHit := MatchCNAMEFingerprints(*cnameEvidence, cnameProviderFingerprints, matcher) + if ShouldEmitCNAME(cnameHit) { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", cnameEvidence.Domain). + Str("provider", cnameHit.Provider). + Str("reason_code", string(cnameHit.ReasonCode)). + Msg("emitting cname finding") + findings = append(findings, model.Finding{ + Domain: cnameEvidence.Domain, + DomainKey: input.DomainKey, + RecordType: model.RecordTypeCNAME, + Target: cnameEvidence.Target, + Provider: cnameHit.Provider, + ReasonCode: string(cnameHit.ReasonCode), + Confidence: ConfidenceForReason(cnameHit.ReasonCode), + }) + } + if cnameHit != nil && !ShouldEmitCNAME(cnameHit) { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", cnameEvidence.Domain). + Str("provider", cnameHit.Provider). + Str("reason_code", string(cnameHit.ReasonCode)). + Msg("cname finding suppressed") + } + } else { + logger.Debug().Str("domain_key", input.DomainKey).Msg("no cname evidence") + } + + nsEvidence := ExtractNSEvidence(input.Results) + if nsEvidence != nil { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", nsEvidence.Domain). + Int("ns_hosts", len(nsEvidence.NSHosts)). + Msg("ns evidence extracted") + nsHit := MatchNSProviderRules(*nsEvidence, nsProviderFingerprints) + if ShouldEmitNSHijack(nsHit) { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", nsEvidence.Domain). + Str("provider", nsHit.Provider). + Str("target", nsHit.Host). + Str("reason_code", string(nsHit.ReasonCode)). + Msg("emitting ns finding") + findings = append(findings, model.Finding{ + Domain: nsEvidence.Domain, + DomainKey: input.DomainKey, + RecordType: model.RecordTypeNS, + Target: nsHit.Host, + Provider: nsHit.Provider, + ReasonCode: string(nsHit.ReasonCode), + Confidence: ConfidenceForReason(nsHit.ReasonCode), + }) + } else if nsHit != nil { + logger.Debug(). + Str("domain_key", input.DomainKey). + Str("domain", nsEvidence.Domain). + Str("provider", nsHit.Provider). + Str("target", nsHit.Host). + Str("reason_code", string(nsHit.ReasonCode)). + Msg("ns finding suppressed") + } + } else { + logger.Debug().Str("domain_key", input.DomainKey).Msg("no ns evidence") + } + + return findings, nil +} diff --git a/scanners/subdomain-takeover/internal/detect/cname_evidence.go b/scanners/subdomain-takeover/internal/detect/cname_evidence.go new file mode 100644 index 0000000000..4466dcd9fc --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/cname_evidence.go @@ -0,0 +1,31 @@ +package detect + +import ( + "strings" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +type CNAMEEvidence struct { + Domain string + Target string + NoResolve bool +} + +func ExtractCNAMEEvidence(results model.ScanResults) *CNAMEEvidence { + if results.CnameRecord == nil || results.Domain == nil { + return nil + } + + return &CNAMEEvidence{ + Domain: *results.Domain, + Target: parseCname(*results.CnameRecord), + NoResolve: len(results.ResolveChain) == 0, + } +} + +func parseCname(record string) string { + normalRecord := normalizeHost(record) + recordTokens := strings.Split(normalRecord, " ") + return recordTokens[len(recordTokens)-1] +} diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules.go b/scanners/subdomain-takeover/internal/detect/cname_rules.go new file mode 100644 index 0000000000..bc1023182e --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/cname_rules.go @@ -0,0 +1,79 @@ +package detect + +import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + +type CNAMEHit struct { + Matched bool + Provider string + ReasonCode ReasonCode + NeedsNX bool +} + +func MatchCNAMEFingerprints(evidence CNAMEEvidence, providerFingerprints []fingerprints.CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { + for _, fp := range providerFingerprints { + if fp.ContainsTarget(evidence.Target) { + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("target", evidence.Target). + Str("provider", fp.Name). + Bool("needs_nx", fp.Nxdomain). + Msg("cname provider target matched") + + hit := CNAMEHit{ + Matched: false, + Provider: fp.Name, + ReasonCode: "", + NeedsNX: fp.Nxdomain, + } + + if hit.NeedsNX { + hit.ReasonCode = ReasonCNAMETargetMatchMissingNXDOMAIN + if evidence.NoResolve { + hit.Matched = true + hit.ReasonCode = ReasonCNAMEDanglingNXDOMAIN + } + } else { + hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP + mode := fingerprints.NormalizeMode(fp.Mode, fp.Fingerprint) + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("fingerprint_mode", string(mode)). + Msg("checking cname body fingerprint") + if matcher != nil && matcher.Contains(evidence.Domain, fp.Fingerprint, mode) { + hit.Matched = true + hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("reason_code", string(hit.ReasonCode)). + Msg("cname body fingerprint matched") + } else { + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("reason_code", string(hit.ReasonCode)). + Msg("cname finding suppressed: body fingerprint missing") + } + } + + if hit.NeedsNX && !hit.Matched { + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("provider", fp.Name). + Str("reason_code", string(hit.ReasonCode)). + Msg("cname finding suppressed: required nxdomain signal missing") + } + + return &hit + } + } + return nil +} + +func ShouldEmitCNAME(hit *CNAMEHit) bool { + if hit == nil { + return false + } + return hit.Matched +} diff --git a/scanners/subdomain-takeover/internal/detect/confidence.go b/scanners/subdomain-takeover/internal/detect/confidence.go new file mode 100644 index 0000000000..06fcd68e53 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/confidence.go @@ -0,0 +1,37 @@ +package detect + +const ( + ConfidenceSuspected = "suspected" + ConfidenceProbable = "probable" + ConfidenceConfirmed = "confirmed" +) + +// ConfidenceForReason maps stable reason codes to confidence levels. +// Unknown reason codes default to suspected. +func ConfidenceForReason(reasonCode ReasonCode) string { + switch reasonCode { + case ReasonCNAMEDanglingNXDOMAIN: + return ConfidenceProbable + case ReasonCNAMEProviderFingerprintBodyMatch: + return ConfidenceProbable + case ReasonCNAMETargetMatchMissingNXDOMAIN: + return ConfidenceSuspected + case ReasonCNAMETargetMatchMissingBodyFP: + return ConfidenceSuspected + + case ReasonNSFullLameProviderVulnerable: + return ConfidenceConfirmed + case ReasonNSPartialLameProviderVulnerable: + return ConfidenceProbable + case ReasonNSRegistrarProviderMatch: + return ConfidenceSuspected + case ReasonNSRegistrarContextInsufficient: + return ConfidenceSuspected + case ReasonNSLameProviderUnknown: + return ConfidenceSuspected + case ReasonNSProviderMatchOnly: + return ConfidenceSuspected + default: + return ConfidenceSuspected + } +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_source.go b/scanners/subdomain-takeover/internal/detect/fingerprint_source.go new file mode 100644 index 0000000000..b6b202e1d7 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_source.go @@ -0,0 +1,18 @@ +package detect + +import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + +type FingerprintSource interface { + CNAME() []fingerprints.CNAMEProviderFingerprint + NS() []fingerprints.NSProviderFingerprint +} + +type GlobalFingerprintSource struct{} + +func (GlobalFingerprintSource) CNAME() []fingerprints.CNAMEProviderFingerprint { + return fingerprints.CNAME() +} + +func (GlobalFingerprintSource) NS() []fingerprints.NSProviderFingerprint { + return fingerprints.NS() +} diff --git a/scanners/subdomain-takeover/internal/detect/logger.go b/scanners/subdomain-takeover/internal/detect/logger.go new file mode 100644 index 0000000000..c3d4a30293 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/logger.go @@ -0,0 +1,9 @@ +package detect + +import "github.com/rs/zerolog" + +var detectLogger = zerolog.Nop() + +func SetLogger(logger zerolog.Logger) { + detectLogger = logger.With().Str("component", "detect").Logger() +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_evidence.go b/scanners/subdomain-takeover/internal/detect/ns_evidence.go new file mode 100644 index 0000000000..7115fdbe2d --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_evidence.go @@ -0,0 +1,43 @@ +package detect + +import ( + "slices" + "strings" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +type NSEvidence struct { + Domain string + NSHosts []string + NSDelegations model.NsDelegations + Registrar *model.RegistrarContext +} + +func ExtractNSEvidence(input model.ScanResults) *NSEvidence { + nsDelegations := input.NsDelegations + if input.Domain == nil || nsDelegations == nil || len(nsDelegations.Hosts) == 0 { + return nil + } + + return &NSEvidence{ + Domain: *input.Domain, + NSHosts: parseHostnames(input.NsDelegations.Hosts), + NSDelegations: *input.NsDelegations, + Registrar: input.RegistrarContext, + } +} + +func parseHostnames(hosts []string) []string { + normalizedHosts := make([]string, 0, len(hosts)) + for _, host := range hosts { + normalizedHosts = append(normalizedHosts, normalizeHost(host)) + } + + slices.Sort(normalizedHosts) + return slices.Compact(normalizedHosts) +} + +func normalizeHost(host string) string { + return strings.Trim(strings.ToLower(host), ".") +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go new file mode 100644 index 0000000000..bc0a672aca --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go @@ -0,0 +1,95 @@ +package detect + +import ( + "strings" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +func getNSHijackReasonCode(lameType string, fingerprint fingerprints.NSProviderFingerprint, registrar *model.RegistrarContext) ReasonCode { + if !isLameType(lameType) { + return ReasonNSProviderMatchOnly + } + + if !isExploitableProviderStatus(fingerprint.Status) { + return ReasonNSLameProviderUnknown + } + + registrarReason := getRegistrarReasonCode(fingerprint.Name, registrar) + if registrarReason != "" { + return registrarReason + } + + switch normalizeLameType(lameType) { + case "full": + return ReasonNSFullLameProviderVulnerable + case "partial": + return ReasonNSPartialLameProviderVulnerable + default: + return ReasonNSProviderMatchOnly + } +} + +func getRegistrarReasonCode(provider string, registrar *model.RegistrarContext) ReasonCode { + if registrar == nil { + return ReasonNSRegistrarContextInsufficient + } + + if !registrar.LookupSuccess || registrar.RegistrarName == "" { + return ReasonNSRegistrarContextInsufficient + } + + if registrar.DelegationMatchesRDAP != nil && !*registrar.DelegationMatchesRDAP { + return ReasonNSRegistrarContextInsufficient + } + + registrarKey := canonicalProviderKey(registrar.RegistrarName) + providerKey := canonicalProviderKey(provider) + + if registrarKey == "" || providerKey == "" { + return ReasonNSRegistrarContextInsufficient + } + + if registrarKey == providerKey { + return ReasonNSRegistrarProviderMatch + } + + return "" +} + +func isRegistrarMismatch(provider string, registrar *model.RegistrarContext) bool { + return getRegistrarReasonCode(provider, registrar) == "" +} + +func isExploitableProviderStatus(status fingerprints.NSProviderStatus) bool { + return status == fingerprints.NSStatusVulnerable || status == fingerprints.NSStatusVulnerableWithPurchase +} + +func isLameType(lameType string) bool { + ltNorm := normalizeLameType(lameType) + return ltNorm == "full" || ltNorm == "partial" +} + +func normalizeLameType(lameType string) string { + return strings.ToLower(strings.TrimSpace(lameType)) +} + +func nsReasonRank(reasonCode ReasonCode) int { + switch reasonCode { + case ReasonNSFullLameProviderVulnerable: + return 4 + case ReasonNSPartialLameProviderVulnerable: + return 3 + case ReasonNSRegistrarProviderMatch: + return 2 + case ReasonNSRegistrarContextInsufficient: + return 2 + case ReasonNSLameProviderUnknown: + return 1 + case ReasonNSProviderMatchOnly: + return 0 + default: + return 0 + } +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules.go b/scanners/subdomain-takeover/internal/detect/ns_rules.go new file mode 100644 index 0000000000..32de3049e3 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_rules.go @@ -0,0 +1,75 @@ +package detect + +import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + +type NSHit struct { + Matched bool + Host string + Provider string + ReasonCode ReasonCode +} + +func MatchNSProviderRules(evidence NSEvidence, providerFingerprints []fingerprints.NSProviderFingerprint) *NSHit { + if len(evidence.NSHosts) == 0 || len(providerFingerprints) == 0 { + detectLogger.Debug().Int("ns_hosts", len(evidence.NSHosts)).Int("fingerprints", len(providerFingerprints)).Msg("skipping ns matching due to insufficient inputs") + return nil + } + + lameType := normalizeLameType(evidence.NSDelegations.Delegation.LameType) + var best *NSHit + + for _, host := range evidence.NSHosts { + for _, fp := range providerFingerprints { + if fp.ContainsNSHost(host) { + hit := &NSHit{ + Matched: false, + Host: host, + Provider: fp.Name, + ReasonCode: getNSHijackReasonCode(lameType, fp, evidence.Registrar), + } + + hit.Matched = hit.ReasonCode == ReasonNSFullLameProviderVulnerable || hit.ReasonCode == ReasonNSPartialLameProviderVulnerable + + rank := nsReasonRank(hit.ReasonCode) + + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("host", host). + Str("provider", fp.Name). + Str("provider_status", string(fp.Status)). + Str("lame_type", lameType). + Str("reason_code", string(hit.ReasonCode)). + Int("rank", rank). + Bool("emittable", hit.Matched). + Msg("ns candidate evaluated") + + if best == nil || rank > nsReasonRank(best.ReasonCode) { + best = hit + } + } + } + } + + if best == nil { + detectLogger.Debug().Str("domain", evidence.Domain).Msg("no ns provider match") + return nil + } + + detectLogger.Debug(). + Str("domain", evidence.Domain). + Str("host", best.Host). + Str("provider", best.Provider). + Str("reason_code", string(best.ReasonCode)). + Int("rank", nsReasonRank(best.ReasonCode)). + Bool("emittable", best.Matched). + Msg("ns best candidate selected") + + return best +} + +func ShouldEmitNSHijack(hit *NSHit) bool { + if hit == nil { + return false + } + return hit.Matched +} diff --git a/scanners/subdomain-takeover/internal/detect/reason_codes.go b/scanners/subdomain-takeover/internal/detect/reason_codes.go new file mode 100644 index 0000000000..6a79589ded --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/reason_codes.go @@ -0,0 +1,17 @@ +package detect + +type ReasonCode string + +const ( + ReasonCNAMEDanglingNXDOMAIN ReasonCode = "CNAME_DANGLING_NXDOMAIN" + ReasonCNAMEProviderFingerprintBodyMatch ReasonCode = "CNAME_PROVIDER_FINGERPRINT_BODY_MATCH" + ReasonCNAMETargetMatchMissingNXDOMAIN ReasonCode = "CNAME_TARGET_MATCH_MISSING_NXDOMAIN" + ReasonCNAMETargetMatchMissingBodyFP ReasonCode = "CNAME_TARGET_MATCH_MISSING_BODY_FINGERPRINT" + + ReasonNSFullLameProviderVulnerable ReasonCode = "NS_FULL_LAME_PROVIDER_VULNERABLE" + ReasonNSPartialLameProviderVulnerable ReasonCode = "NS_PARTIAL_LAME_PROVIDER_VULNERABLE" + ReasonNSRegistrarProviderMatch ReasonCode = "NS_REGISTRAR_PROVIDER_MATCH" + ReasonNSRegistrarContextInsufficient ReasonCode = "NS_REGISTRAR_CONTEXT_INSUFFICIENT" + ReasonNSLameProviderUnknown ReasonCode = "NS_LAME_PROVIDER_UNKNOWN" + ReasonNSProviderMatchOnly ReasonCode = "NS_PROVIDER_MATCH_ONLY" +) diff --git a/scanners/subdomain-takeover/internal/detect/registrar_helpers.go b/scanners/subdomain-takeover/internal/detect/registrar_helpers.go new file mode 100644 index 0000000000..a31f942744 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/registrar_helpers.go @@ -0,0 +1,139 @@ +package detect + +import "strings" + +func normalizeOrgName(raw string) string { + s := strings.ToLower(strings.TrimSpace(raw)) + if s == "" { + return "" + } + + // Replace punctuation/separators with spaces. + replacer := strings.NewReplacer( + ".", " ", + ",", " ", + ";", " ", + ":", " ", + "(", " ", + ")", " ", + "[", " ", + "]", " ", + "{", " ", + "}", " ", + "/", " ", + "\\", " ", + "-", " ", + "_", " ", + "&", " and ", + "'", "", + "\"", "", + ) + s = replacer.Replace(s) + + // Collapse repeated whitespace. + s = strings.Join(strings.Fields(s), " ") + if s == "" { + return "" + } + + // Remove common legal/company suffixes and generic noise words. + stop := map[string]struct{}{ + "inc": {}, "incorporated": {}, "llc": {}, "l l c": {}, "ltd": {}, "limited": {}, + "corp": {}, "corporation": {}, "co": {}, "company": {}, "gmbh": {}, "ag": {}, + "sa": {}, "s a": {}, "sas": {}, "plc": {}, "pte": {}, "bv": {}, "nv": {}, + "the": {}, "group": {}, "holdings": {}, "technologies": {}, "technology": {}, + "services": {}, "service": {}, + } + + tokens := strings.Fields(s) + out := make([]string, 0, len(tokens)) + for _, t := range tokens { + if _, drop := stop[t]; drop { + continue + } + out = append(out, t) + } + + return strings.Join(out, " ") +} +func canonicalProviderKey(raw string) string { + + var canonicalProviderAliases = map[string][]string{ + "aws-route53": { + "aws route 53", "route 53", "route53", "amazon route 53", "amazon web services", "aws", + }, + "azure-dns": { + "azure", "azure dns", "microsoft", "microsoft azure", + }, + "cloudflare": { + "cloudflare", "cloudflare inc", + }, + "digitalocean": { + "digital ocean", "digitalocean", + }, + "dnsmadeeasy": { + "dns made easy", "dnsmadeeasy", + }, + "dnsimple": { + "dnsimple", + }, + "domaincom": { + "domain com", "domain.com", "domain", + }, + "dreamhost": { + "dreamhost", + }, + "easydns": { + "easydns", "easy dns", + }, + "gandi": { + "gandi", "gandi sas", "gandi.net", + }, + "google-cloud-dns": { + "google cloud", "google cloud dns", "google domains", "googledomains", "google", + }, + "hurricane-electric": { + "hurricane electric", "he net", "he", + }, + "linode": { + "linode", "akamai linode", "akamai", + }, + "namecom": { + "name.com", "name com", "namecom", + }, + "namecheap": { + "namecheap", + }, + "network-solutions": { + "network solutions", "web.com", "webcom", + }, + "ns1": { + "ns1", "nsone", "nsone.net", + }, + "reg-ru": { + "reg ru", "reg.ru", + }, + "tierranet": { + "tierranet", "tierra net", "domaindiscover", + }, + "ultradns": { + "ultradns", "neustar ultradns", "neustar", + }, + "yahoo-smb": { + "yahoo small business", "yahoo", "yns", + }, + } + + s := normalizeOrgName(raw) // lowercase, trim, collapse spaces, strip punctuation/legal suffixes + if s == "" { + return "" + } + for key, aliases := range canonicalProviderAliases { + for _, a := range aliases { + if s == a || strings.Contains(s, a) { + return key + } + } + } + return "" +} From 310dececfed1bbdf23ff9d1ccda8e563b92277d5 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 4 Aug 2026 14:19:56 -0300 Subject: [PATCH 48/65] subdomain-takeover: add fingerprint package and datasets --- .../fingerprints/data/cname_fingerprints.json | 227 ++++++++++ .../fingerprints/data/ns_fingerprints.json | 416 ++++++++++++++++++ .../internal/fingerprints/loader.go | 128 ++++++ .../internal/fingerprints/types.go | 105 +++++ 4 files changed, 876 insertions(+) create mode 100644 scanners/subdomain-takeover/internal/fingerprints/data/cname_fingerprints.json create mode 100644 scanners/subdomain-takeover/internal/fingerprints/data/ns_fingerprints.json create mode 100644 scanners/subdomain-takeover/internal/fingerprints/loader.go create mode 100644 scanners/subdomain-takeover/internal/fingerprints/types.go diff --git a/scanners/subdomain-takeover/internal/fingerprints/data/cname_fingerprints.json b/scanners/subdomain-takeover/internal/fingerprints/data/cname_fingerprints.json new file mode 100644 index 0000000000..cbf3aa24be --- /dev/null +++ b/scanners/subdomain-takeover/internal/fingerprints/data/cname_fingerprints.json @@ -0,0 +1,227 @@ +[ + { + "cname": [ + "elasticbeanstalk.com" + ], + "name": "AWS/Elastic Beanstalk", + "nxdomain": true, + "fingerprint": "NXDOMAIN" + }, + { + "cname": [ + "s3.amazonaws.com" + ], + "name": "AWS/S3", + "nxdomain": false, + "fingerprint": "The specified bucket does not exist" + }, + { + "cname": [ + "agilecrm.com" + ], + "name": "Agile CRM", + "nxdomain": false, + "fingerprint": "Sorry, this page is no longer available." + }, + { + "cname": [ + "airee.ru" + ], + "name": "Airee.ru", + "nxdomain": false, + "fingerprint": "Ошибка 402. Сервис Айри.рф не оплачен" + }, + { + "cname": [ + "animaapp.io" + ], + "name": "Anima", + "nxdomain": false, + "fingerprint": "The page you were looking for does not exist." + }, + { + "cname": [ + "bitbucket.io" + ], + "name": "Bitbucket", + "nxdomain": false, + "fingerprint": "Repository not found" + }, + { + "cname": [ + "trydiscourse.com" + ], + "name": "Discourse", + "nxdomain": true, + "fingerprint": "NXDOMAIN" + }, + { + "cname": [ + "furyns.com" + ], + "name": "Gemfury", + "nxdomain": false, + "fingerprint": "404: This page could not be found." + }, + { + "cname": [ + "ghost.io" + ], + "name": "Ghost", + "nxdomain": false, + "fingerprint": "Site unavailable\\.\u0026#124;Failed to resolve DNS path for this host" + }, + { + "cname": [ + "hatenablog.com" + ], + "name": "HatenaBlog", + "nxdomain": false, + "fingerprint": "404 Blog is not found" + }, + { + "cname": [ + "helpjuice.com" + ], + "name": "Help Juice", + "nxdomain": false, + "fingerprint": "We could not find what you're looking for." + }, + { + "cname": [ + "helpscoutdocs.com" + ], + "name": "Help Scout", + "nxdomain": false, + "fingerprint": "No settings were found for this company:" + }, + { + "cname": [ + "helprace.com" + ], + "name": "Helprace", + "nxdomain": false, + "fingerprint": "HTTP_STATUS=301" + }, + { + "cname": [ + "youtrack.cloud" + ], + "name": "JetBrains", + "nxdomain": false, + "fingerprint": "is not a registered InCloud YouTrack" + }, + { + "cname": [ + "launchrock.com" + ], + "name": "LaunchRock", + "nxdomain": false, + "fingerprint": "HTTP_STATUS=500" + }, + { + "cname": [ + "cloudapp.azure.com", + "azurewebsites.net", + "blob.core.windows.net", + "cloudapp.azure.com", + "azure-api.net", + "azurehdinsight.net", + "azureedge.net", + "azurecontainer.io", + "database.windows.net", + "azuredatalakestore.net", + "search.windows.net", + "azurecr.io", + "redis.cache.windows.net", + "azurehdinsight.net", + "servicebus.windows.net", + "trafficmanager.net", + "visualstudio.com" + ], + "name": "Microsoft Azure", + "nxdomain": true, + "fingerprint": "NXDOMAIN" + }, + { + "cname": [ + "ngrok.io" + ], + "name": "Ngrok", + "nxdomain": false, + "fingerprint": "Tunnel .*.ngrok.io not found" + }, + { + "cname": [ + "readme.io" + ], + "name": "Readme.io", + "nxdomain": false, + "fingerprint": "The creators of this project are still working on making everything perfect!" + }, + { + "cname": [ + "52.16.160.97" + ], + "name": "SmartJobBoard", + "nxdomain": false, + "fingerprint": "This job board website is either expired or its domain name is invalid." + }, + { + "cname": [ + "s.strikinglydns.com" + ], + "name": "Strikingly", + "nxdomain": false, + "fingerprint": "PAGE NOT FOUND." + }, + { + "cname": [ + "na-west1.surge.sh" + ], + "name": "Surge.sh", + "nxdomain": false, + "fingerprint": "project not found" + }, + { + "cname": [ + "surveysparrow.com" + ], + "name": "SurveySparrow", + "nxdomain": false, + "fingerprint": "Account not found." + }, + { + "cname": [ + "read.uberflip.com" + ], + "name": "Uberflip", + "nxdomain": false, + "fingerprint": "The URL you've accessed does not provide a hub." + }, + { + "cname": [ + "stats.uptimerobot.com" + ], + "name": "Uptimerobot", + "nxdomain": false, + "fingerprint": "page not found" + }, + { + "cname": [ + "wordpress.com" + ], + "name": "Wordpress", + "nxdomain": false, + "fingerprint": "Do you want to register .*.wordpress.com?" + }, + { + "cname": [ + "worksites.net", + "69.164.223.206" + ], + "name": "Worksites", + "nxdomain": false, + "fingerprint": "Hello! Sorry, but the website you\u0026rsquo;re looking for doesn\u0026rsquo;t exist." + } +] diff --git a/scanners/subdomain-takeover/internal/fingerprints/data/ns_fingerprints.json b/scanners/subdomain-takeover/internal/fingerprints/data/ns_fingerprints.json new file mode 100644 index 0000000000..85fd5262bf --- /dev/null +++ b/scanners/subdomain-takeover/internal/fingerprints/data/ns_fingerprints.json @@ -0,0 +1,416 @@ +[ + { + "name": "000Domains", + "provider_url": "https://000domains.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.000domains.com", + "ns2.000domains.com", + "fwns1.000domains.com", + "fwns2.000domains.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/19" + }, + { + "name": "AWS Route 53", + "provider_url": "https://aws.amazon.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns-*.awsdns-*.org", + "ns-*.awsdns-*.co.uk", + "ns-*.awsdns-*.com", + "ns-*.awsdns-*.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/1" + }, + { + "name": "Azure (Microsoft)", + "provider_url": "https://azure.microsoft.com/", + "status": "edge_case", + "host_patterns": [ + "ns1-*.azure-dns.com", + "ns2-*.azure-dns.net", + "ns3-*.azure-dns.org", + "ns4-*.azure-dns.info" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/5" + }, + { + "name": "BigCommerce", + "provider_url": "https://bigcommerce.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.bigcommerce.com", + "ns2.bigcommerce.com", + "ns3.bigcommerce.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/35" + }, + { + "name": "Bizland", + "provider_url": "https://bizland.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.bizland.com", + "ns2.bizland.com", + "clickme.click2site.com", + "clickme2.click2site.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/3" + }, + { + "name": "ClouDNS", + "provider_url": "https://cloudns.net/", + "status": "not_vulnerable", + "host_patterns": [ + "*.cloudns.net" + ] + }, + { + "name": "Cloudflare", + "provider_url": "https://cloudflare.com/", + "status": "not_vulnerable", + "host_patterns": [ + "*.ns.cloudflare.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/10" + }, + { + "name": "Digital Ocean", + "provider_url": "https://digitalocean.com/", + "status": "vulnerable", + "host_patterns": [ + "ns1.digitalocean.com", + "ns2.digitalocean.com", + "ns3.digitalocean.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/22" + }, + { + "name": "DNSMadeEasy", + "provider_url": "https://dnsmadeeasy.com/", + "status": "vulnerable", + "host_patterns": [ + "ns*.dnsmadeeasy.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/6" + }, + { + "name": "DNSimple", + "provider_url": "https://dnsimple.com/", + "status": "vulnerable", + "host_patterns": [ + "ns1.dnsimple.com", + "ns2.dnsimple.com", + "ns3.dnsimple.com", + "ns4.dnsimple.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/16" + }, + { + "name": "Domain.com", + "provider_url": "https://domain.com/", + "status": "vulnerable_with_purchase", + "host_patterns": [ + "ns1.domain.com", + "ns2.domain.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/17" + }, + { + "name": "DomainPeople", + "provider_url": "https://domainpeople.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.domainpeople.com", + "ns2.domainpeople.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/14" + }, + { + "name": "Dotster", + "provider_url": "https://dotster.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.dotster.com", + "ns2.dotster.com", + "ns1.nameresolve.com", + "ns2.nameresolve.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/18" + }, + { + "name": "Dreamhost", + "provider_url": "https://dreamhost.com/", + "status": "edge_case", + "host_patterns": [ + "ns1.dreamhost.com", + "ns2.dreamhost.com", + "ns3.dreamhost.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/40" + }, + { + "name": "EasyDNS", + "provider_url": "https://easydns.com/", + "status": "not_vulnerable", + "host_patterns": [ + "dns1.easydns.com", + "dns2.easydns.net", + "dns3.easydns.org", + "dns4.easydns.info" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/9" + }, + { + "name": "Gandi.net", + "provider_url": "https://gandi.net/", + "status": "not_vulnerable", + "host_patterns": [ + "a.dns.gandi.net", + "b.dns.gandi.net", + "c.dns.gandi.net" + ] + }, + { + "name": "Google Cloud", + "provider_url": "https://cloud.google.com/", + "status": "edge_case", + "host_patterns": [ + "ns-cloud-*.googledomains.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/2" + }, + { + "name": "Hostinger (old NS)", + "provider_url": "https://hostinger.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.hostinger.com", + "ns2.hostinger.com" + ] + }, + { + "name": "Hover", + "provider_url": "https://hover.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.hover.com", + "ns2.hover.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/21" + }, + { + "name": "Hurricane Electric", + "provider_url": "https://dns.he.net/", + "status": "vulnerable", + "host_patterns": [ + "ns1.he.net", + "ns2.he.net", + "ns3.he.net", + "ns4.he.net", + "ns5.he.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/25" + }, + { + "name": "Linode", + "provider_url": "https://linode.com/", + "status": "vulnerable", + "host_patterns": [ + "ns1.linode.com", + "ns2.linode.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/26" + }, + { + "name": "MediaTemple (mt)", + "provider_url": "https://mediatemple.net/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.mediatemple.net", + "ns2.mediatemple.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/23" + }, + { + "name": "MyDomain", + "provider_url": "https://mydomain.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns1.mydomain.com", + "ns2.mydomain.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/4" + }, + { + "name": "Name.com", + "provider_url": "https://name.com/", + "status": "vulnerable_with_purchase", + "host_patterns": [ + "ns1*.name.com", + "ns2*.name.com", + "ns3*.name.com", + "ns4*.name.com" + ], + "instructions_url": "https://github.com/libertalialtd/can-i-take-over-dns/issues/8" + }, + { + "name": "Namecheap", + "provider_url": "https://namecheap.com/", + "status": "not_vulnerable", + "host_patterns": [ + "*.namecheaphosting.com", + "*.registrar-servers.com" + ] + }, + { + "name": "Network Solutions", + "provider_url": "https://networksolutions.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.worldnic.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/15" + }, + { + "name": "NS1", + "provider_url": "https://nsone.net/", + "status": "registration_closed", + "host_patterns": [ + "dns1.p*.nsone.net", + "dns2.p*.nsone.net", + "dns3.p*.nsone.net", + "dns4.p*.nsone.net" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/7" + }, + { + "name": "TierraNet", + "provider_url": "https://tierra.net/", + "status": "vulnerable", + "host_patterns": [ + "ns1.domaindiscover.com", + "ns2.domaindiscover.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/24" + }, + { + "name": "Reg.ru", + "provider_url": "https://reg.ru/", + "status": "vulnerable", + "host_patterns": [ + "ns1.reg.ru", + "ns2.reg.ru" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/28" + }, + { + "name": "UltraDNS", + "provider_url": "https://www.home.neustar/dns-services/ultra-dns", + "status": "not_vulnerable", + "host_patterns": [ + "pdns*.ultradns.com", + "udns*.ultradns.com", + "sdns*.ultradns.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/29" + }, + { + "name": "Yahoo Small Business", + "provider_url": "https://yahoosmallbusiness.com/", + "status": "vulnerable_with_purchase", + "host_patterns": [ + "yns1.yahoo.com", + "yns2.yahoo.com" + ], + "instructions_url": "https://github.com/indianajson/can-i-take-over-dns/issues/20" + }, + { + "name": "Activision", + "provider_url": "https://activision.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.activision.com" + ], + "private_dns": true + }, + { + "name": "Adobe", + "provider_url": "https://adobe.com/", + "status": "not_vulnerable", + "host_patterns": [ + "adobe-dns-0*.adobe.com" + ], + "private_dns": true + }, + { + "name": "Apple", + "provider_url": "https://apple.com/", + "status": "not_vulnerable", + "host_patterns": [ + "a.ns.apple.com", + "b.ns.apple.com", + "c.ns.apple.com", + "d.ns.apple.com" + ], + "private_dns": true + }, + { + "name": "Automattic", + "provider_url": "https://automattic.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.automattic.com" + ], + "private_dns": true + }, + { + "name": "Capital One", + "provider_url": "https://capitalone.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.capitalone.com" + ], + "private_dns": true + }, + { + "name": "Disney", + "provider_url": "https://disney.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.twdcns.com", + "ns*.twdcns.info", + "ns*.twdcns.co.uk" + ], + "private_dns": true + }, + { + "name": "Google", + "provider_url": "https://google.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns*.google.com" + ], + "private_dns": true + }, + { + "name": "Lowe's", + "provider_url": "https://lowes.com/", + "status": "not_vulnerable", + "host_patterns": [ + "authns*.lowes.com" + ], + "private_dns": true + }, + { + "name": "T-Mobile", + "provider_url": "https://tmobileus.com/", + "status": "not_vulnerable", + "host_patterns": [ + "ns10.tmobileus.com", + "ns10.tmobileus.net" + ], + "private_dns": true + } +] diff --git a/scanners/subdomain-takeover/internal/fingerprints/loader.go b/scanners/subdomain-takeover/internal/fingerprints/loader.go new file mode 100644 index 0000000000..9bcec10792 --- /dev/null +++ b/scanners/subdomain-takeover/internal/fingerprints/loader.go @@ -0,0 +1,128 @@ +package fingerprints + +import ( + "embed" + "encoding/json" + "fmt" + "strings" + "sync" + + "github.com/rs/zerolog" +) + +//go:embed data/*.json +var dataFS embed.FS + +var ( + loadOnce sync.Once + loadErr error + + cnameProviderFingerprints []CNAMEProviderFingerprint + nsProviderFingerprints []NSProviderFingerprint +) + +func Load(logger zerolog.Logger) error { + log := logger.With().Str("component", "fingerprint_loader").Logger() + + loadOnce.Do(func() { + var cname []CNAMEProviderFingerprint + var ns []NSProviderFingerprint + + if err := loadJSON("data/cname_fingerprints.json", &cname); err != nil { + log.Error().Err(err).Str("dataset", "cname_fingerprints").Msg("failed to load fingerprint dataset") + loadErr = err + return + } + + if err := loadJSON("data/ns_fingerprints.json", &ns); err != nil { + log.Error().Err(err).Str("dataset", "ns_fingerprints").Msg("failed to load fingerprint dataset") + loadErr = err + return + } + + if err := validateCNAMEFingerprints(cname); err != nil { + log.Error().Err(err).Str("dataset", "cname_fingerprints").Msg("invalid fingerprint dataset") + loadErr = err + return + } + + if err := validateNSFingerprints(ns); err != nil { + log.Error().Err(err).Str("dataset", "ns_fingerprints").Msg("invalid fingerprint dataset") + loadErr = err + return + } + + cnameProviderFingerprints = cname + nsProviderFingerprints = ns + + log.Info().Int("cname_fingerprints", len(cname)).Int("ns_fingerprints", len(ns)).Msg("fingerprint datasets loaded") + }) + + return loadErr +} + +func CNAME() []CNAMEProviderFingerprint { + return cnameProviderFingerprints +} + +func NS() []NSProviderFingerprint { + return nsProviderFingerprints +} + +func loadJSON(path string, out any) error { + b, err := dataFS.ReadFile(path) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + + if err := json.Unmarshal(b, out); err != nil { + return fmt.Errorf("decode %s: %w", path, err) + } + + return nil +} + +func validateCNAMEFingerprints(fingerprints []CNAMEProviderFingerprint) error { + for i, fp := range fingerprints { + if strings.TrimSpace(fp.Name) == "" { + return fmt.Errorf("cname fingerprint[%d] missing name", i) + } + if len(fp.Cname) == 0 { + return fmt.Errorf("cname fingerprint[%d] has no cname patterns", i) + } + + for j, cname := range fp.Cname { + if strings.TrimSpace(cname) == "" { + return fmt.Errorf("cname fingerprint[%d] has empty cname pattern at index %d", i, j) + } + } + + if strings.TrimSpace(fp.Fingerprint) == "" { + return fmt.Errorf("cname fingerprint[%d] missing fingerprint", i) + } + + fingerprints[i].Mode = NormalizeMode(fp.Mode, fp.Fingerprint) + } + + return nil +} + +func validateNSFingerprints(fingerprints []NSProviderFingerprint) error { + for i, fp := range fingerprints { + if strings.TrimSpace(fp.Name) == "" { + return fmt.Errorf("ns fingerprint[%d] missing name", i) + } + + if len(fp.HostPatterns) == 0 { + return fmt.Errorf("ns fingerprint[%d] has no host patterns", i) + } + + for j, pattern := range fp.HostPatterns { + if strings.TrimSpace(pattern) == "" { + return fmt.Errorf("ns fingerprint[%d] has empty host pattern at index %d", i, j) + } + } + } + + return nil +} diff --git a/scanners/subdomain-takeover/internal/fingerprints/types.go b/scanners/subdomain-takeover/internal/fingerprints/types.go new file mode 100644 index 0000000000..ac275d8430 --- /dev/null +++ b/scanners/subdomain-takeover/internal/fingerprints/types.go @@ -0,0 +1,105 @@ +package fingerprints + +import "strings" + +type CNAMEProviderFingerprint struct { + Cname []string `json:"cname"` + Name string `json:"name"` + Nxdomain bool `json:"nxdomain"` + Fingerprint string `json:"fingerprint"` + Mode FingerprintMode `json:"mode,omitempty"` +} + +type FingerprintMode string + +const ( + FingerprintModeLiteral FingerprintMode = "literal" + FingerprintModeRegex FingerprintMode = "regex" +) + +type NSProviderStatus string + +const ( + NSStatusVulnerable NSProviderStatus = "vulnerable" + NSStatusNotVulnerable NSProviderStatus = "not_vulnerable" + NSStatusEdgeCase NSProviderStatus = "edge_case" + NSStatusVulnerableWithPurchase NSProviderStatus = "vulnerable_with_purchase" + NSStatusRegistrationClosed NSProviderStatus = "registration_closed" +) + +type NSProviderFingerprint struct { + Name string `json:"name"` + ProviderURL string `json:"provider_url"` + Status NSProviderStatus `json:"status"` + HostPatterns []string `json:"host_patterns"` + InstructionsURL string `json:"instructions_url,omitempty"` + PrivateDNS bool `json:"private_dns,omitempty"` +} + +func (f *NSProviderFingerprint) ContainsNSHost(host string) bool { + host = strings.ToLower(strings.TrimSuffix(host, ".")) + for _, pattern := range f.HostPatterns { + if wildcardHostMatch(pattern, host) { + return true + } + } + return false +} + +func wildcardHostMatch(pattern, host string) bool { + pattern = strings.ToLower(strings.TrimSuffix(pattern, ".")) + if !strings.Contains(pattern, "*") { + return host == pattern + } + + parts := strings.Split(pattern, "*") + if len(parts) == 2 { + return strings.HasPrefix(host, parts[0]) && strings.HasSuffix(host, parts[1]) + } + + idx := 0 + for i, part := range parts { + if part == "" { + continue + } + + pos := strings.Index(host[idx:], part) + if pos < 0 { + return false + } + + if i == 0 && !strings.HasPrefix(host, part) { + return false + } + + idx += pos + len(part) + } + + last := parts[len(parts)-1] + if last != "" && !strings.HasSuffix(host, last) { + return false + } + + return true +} + +func (f *CNAMEProviderFingerprint) ContainsTarget(target string) bool { + for _, cname := range f.Cname { + if strings.HasSuffix(target, cname) { + return true + } + } + return false +} + +func NormalizeMode(mode FingerprintMode, fingerprint string) FingerprintMode { + if mode == FingerprintModeLiteral || mode == FingerprintModeRegex { + return mode + } + + if strings.Contains(fingerprint, ".*") || strings.Contains(fingerprint, "\\") { + return FingerprintModeRegex + } + + return FingerprintModeLiteral +} From 145715e2d4c3a00c3d98aae3ca71f4cd977b4716 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 4 Aug 2026 14:19:58 -0300 Subject: [PATCH 49/65] subdomain-takeover: add unit tests --- .../internal/app/runner_test.go | 130 +++++++++++ .../internal/app/worker_test.go | 205 ++++++++++++++++++ .../internal/detect/body_matcher_test.go | 65 ++++++ .../internal/detect/classifier_test.go | 139 ++++++++++++ .../internal/detect/cname_rules_test.go | 106 +++++++++ .../internal/detect/confidence_test.go | 26 +++ .../internal/detect/evidence_test.go | 128 +++++++++++ .../detect/fingerprint_source_test.go | 22 ++ .../internal/detect/logger_test.go | 15 ++ .../internal/detect/ns_reasoning_test.go | 121 +++++++++++ .../internal/detect/ns_rules_test.go | 160 ++++++++++++++ .../internal/detect/test_helpers_test.go | 33 +++ .../internal/fingerprints/loader_test.go | 94 ++++++++ .../internal/fingerprints/types_test.go | 47 ++++ .../internal/messaging/health_test.go | 52 +++++ .../internal/messaging/publisher_test.go | 113 ++++++++++ 16 files changed, 1456 insertions(+) create mode 100644 scanners/subdomain-takeover/internal/app/runner_test.go create mode 100644 scanners/subdomain-takeover/internal/app/worker_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/body_matcher_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/classifier_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/cname_rules_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/confidence_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/evidence_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/logger_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/ns_rules_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/test_helpers_test.go create mode 100644 scanners/subdomain-takeover/internal/fingerprints/loader_test.go create mode 100644 scanners/subdomain-takeover/internal/fingerprints/types_test.go create mode 100644 scanners/subdomain-takeover/internal/messaging/health_test.go create mode 100644 scanners/subdomain-takeover/internal/messaging/publisher_test.go diff --git a/scanners/subdomain-takeover/internal/app/runner_test.go b/scanners/subdomain-takeover/internal/app/runner_test.go new file mode 100644 index 0000000000..b732ecd76a --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/runner_test.go @@ -0,0 +1,130 @@ +package app + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type fakeMessagesIter struct { + mu sync.Mutex + msgs []jetstream.Msg + err error + idx int + stopCalled bool +} + +func (f *fakeMessagesIter) Next(...jetstream.NextOpt) (jetstream.Msg, error) { + f.mu.Lock() + defer f.mu.Unlock() + + if f.err != nil { + return nil, f.err + } + if f.idx >= len(f.msgs) { + return nil, jetstream.ErrMsgIteratorClosed + } + m := f.msgs[f.idx] + f.idx++ + return m, nil +} + +func (f *fakeMessagesIter) Stop() { f.stopCalled = true } +func (f *fakeMessagesIter) Drain() {} + +type fakeHandler struct { + mu sync.Mutex + count int +} + +func (f *fakeHandler) Handle(context.Context, jetstream.Msg) error { + f.mu.Lock() + f.count++ + f.mu.Unlock() + return nil +} + +// nolint:revive // nats Conn fields are unexported; this fake nil connection path only. +func TestRun_ReturnsImmediatelyWhenConnectionUnhealthy(t *testing.T) { + iter := &fakeMessagesIter{} + h := &fakeHandler{} + + deps := RunnerDeps{ + Logger: zerolog.Nop(), + WorkerCount: 2, + Iter: iter, + Worker: h, + NC: nil, + } + + Run(context.Background(), deps) + + if h.count != 0 { + t.Fatalf("expected no handled messages, got %d", h.count) + } +} + +func TestRun_ClampsWorkerCountBelowOne(t *testing.T) { + origCheckConnection := checkConnection + t.Cleanup(func() { checkConnection = origCheckConnection }) + + checkCalls := 0 + checkConnection = func(_ *nats.Conn) error { + checkCalls++ + if checkCalls > 1 { + return errors.New("stop") + } + return nil + } + + iter := &fakeMessagesIter{msgs: []jetstream.Msg{&fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"}}} + h := &fakeHandler{} + + deps := RunnerDeps{ + Logger: zerolog.Nop(), + WorkerCount: 0, + Iter: iter, + Worker: h, + NC: nil, + } + + Run(context.Background(), deps) + if h.count != 1 { + t.Fatalf("expected one handled message, got %d", h.count) + } +} + +func TestRun_ExitsWhenContextCancelledDuringNextErrors(t *testing.T) { + origCheckConnection := checkConnection + t.Cleanup(func() { checkConnection = origCheckConnection }) + checkConnection = func(_ *nats.Conn) error { return nil } + + iter := &fakeMessagesIter{err: errors.New("next failed")} + h := &fakeHandler{} + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(5 * time.Millisecond) + cancel() + }() + + deps := RunnerDeps{ + Logger: zerolog.Nop(), + WorkerCount: 1, + Iter: iter, + Worker: h, + NC: nil, + } + + Run(ctx, deps) + + if h.count != 0 { + t.Fatalf("expected no handled messages, got %d", h.count) + } +} diff --git a/scanners/subdomain-takeover/internal/app/worker_test.go b/scanners/subdomain-takeover/internal/app/worker_test.go new file mode 100644 index 0000000000..fc771e4e15 --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/worker_test.go @@ -0,0 +1,205 @@ +package app + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type fakePublisher struct { + err error + published []model.Finding +} + +func (f *fakePublisher) Publish(_ context.Context, finding model.Finding) error { + if f.err != nil { + return f.err + } + f.published = append(f.published, finding) + return nil +} + +type fakeClassifier struct { + findings []model.Finding + err error +} + +func (f fakeClassifier) Classify(model.Input) ([]model.Finding, error) { + if f.err != nil { + return nil, f.err + } + return f.findings, nil +} + +type fakeJSMsg struct { + data []byte + subject string + + ackErr error + nakErr error + termErr error + + ackCount int + nakCount int + termCount int +} + +func (m *fakeJSMsg) Metadata() (*jetstream.MsgMetadata, error) { return nil, nil } +func (m *fakeJSMsg) Data() []byte { return m.data } +func (m *fakeJSMsg) Headers() nats.Header { return nil } +func (m *fakeJSMsg) Subject() string { return m.subject } +func (m *fakeJSMsg) Reply() string { return "" } + +func (m *fakeJSMsg) Ack() error { + m.ackCount++ + return m.ackErr +} + +func (m *fakeJSMsg) DoubleAck(context.Context) error { return nil } + +func (m *fakeJSMsg) Nak() error { + m.nakCount++ + return m.nakErr +} + +func (m *fakeJSMsg) NakWithDelay(time.Duration) error { return nil } +func (m *fakeJSMsg) InProgress() error { return nil } + +func (m *fakeJSMsg) Term() error { + m.termCount++ + return m.termErr +} + +func (m *fakeJSMsg) TermWithReason(string) error { return nil } + +func TestWorkerHandle(t *testing.T) { + logger := zerolog.Nop() + + t.Run("decode error terminates message", func(t *testing.T) { + pub := &fakePublisher{} + classifier := fakeClassifier{} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{data: []byte("{not-json"), subject: "scans.dns_scanner_results"} + err := worker.Handle(context.Background(), msg) + if err == nil { + t.Fatal("expected decode error") + } + if msg.termCount != 1 { + t.Fatalf("expected term once, got %d", msg.termCount) + } + if msg.nakCount != 0 || msg.ackCount != 0 { + t.Fatalf("unexpected ack/nak counts: ack=%d nak=%d", msg.ackCount, msg.nakCount) + } + }) + + t.Run("classification error naks message", func(t *testing.T) { + pub := &fakePublisher{} + classifier := fakeClassifier{err: errors.New("classify failed")} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"} + err := worker.Handle(context.Background(), msg) + if err == nil { + t.Fatal("expected classification error") + } + if msg.nakCount != 1 { + t.Fatalf("expected nak once, got %d", msg.nakCount) + } + if msg.ackCount != 0 || msg.termCount != 0 { + t.Fatalf("unexpected ack/term counts: ack=%d term=%d", msg.ackCount, msg.termCount) + } + }) + + t.Run("publish error naks message", func(t *testing.T) { + pub := &fakePublisher{err: errors.New("publish failed")} + classifier := fakeClassifier{findings: []model.Finding{{Domain: "a.example.ca"}}} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"} + err := worker.Handle(context.Background(), msg) + if err == nil { + t.Fatal("expected publish error") + } + if msg.nakCount != 1 { + t.Fatalf("expected nak once, got %d", msg.nakCount) + } + if msg.ackCount != 0 || msg.termCount != 0 { + t.Fatalf("unexpected ack/term counts: ack=%d term=%d", msg.ackCount, msg.termCount) + } + }) + + t.Run("successful processing publishes all findings and acks", func(t *testing.T) { + pub := &fakePublisher{} + classifier := fakeClassifier{findings: []model.Finding{{Domain: "a.example.ca"}, {Domain: "b.example.ca"}}} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"} + err := worker.Handle(context.Background(), msg) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(pub.published) != 2 { + t.Fatalf("expected 2 published findings, got %d", len(pub.published)) + } + if msg.ackCount != 1 { + t.Fatalf("expected ack once, got %d", msg.ackCount) + } + if msg.nakCount != 0 || msg.termCount != 0 { + t.Fatalf("unexpected nak/term counts: nak=%d term=%d", msg.nakCount, msg.termCount) + } + }) + + t.Run("ack failure returns error", func(t *testing.T) { + pub := &fakePublisher{} + classifier := fakeClassifier{} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{ + data: []byte(`{"domain_key":"k","results":{}}`), + subject: "scans.dns_scanner_results", + ackErr: errors.New("ack failed"), + } + + err := worker.Handle(context.Background(), msg) + if err == nil { + t.Fatal("expected ack error") + } + if msg.ackCount != 1 { + t.Fatalf("expected ack once, got %d", msg.ackCount) + } + }) + + t.Run("decode error still returned when term fails", func(t *testing.T) { + pub := &fakePublisher{} + classifier := fakeClassifier{} + worker := NewWorker(logger, pub, classifier) + + msg := &fakeJSMsg{ + data: []byte("{bad-json"), + subject: "scans.dns_scanner_results", + termErr: errors.New("term failed"), + } + + err := worker.Handle(context.Background(), msg) + if err == nil { + t.Fatal("expected decode error") + } + if msg.termCount != 1 { + t.Fatalf("expected term once, got %d", msg.termCount) + } + }) +} + +func TestDecodeScan_TrimsTrailingNewline(t *testing.T) { + _, err := decodeScan([]byte("{\"domain_key\":\"k\",\"results\":{}}\n")) + if err != nil { + t.Fatalf("expected decode success, got error: %v", err) + } +} diff --git a/scanners/subdomain-takeover/internal/detect/body_matcher_test.go b/scanners/subdomain-takeover/internal/detect/body_matcher_test.go new file mode 100644 index 0000000000..59df870656 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/body_matcher_test.go @@ -0,0 +1,65 @@ +package detect + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" +) + +func TestNoopBodyFingerprintMatcher(t *testing.T) { + matcher := NewNoopBodyFingerprintMatcher() + if matcher.Contains("example.ca", "anything", fingerprints.FingerprintModeLiteral) { + t.Fatal("noop matcher should never match") + } +} + +func TestHTTPBodyFingerprintMatcher(t *testing.T) { + t.Run("returns false when matcher/client nil", func(t *testing.T) { + var matcher *HTTPBodyFingerprintMatcher + if matcher.Contains("example.ca", "x", fingerprints.FingerprintModeLiteral) { + t.Fatal("expected false for nil matcher") + } + + matcher = &HTTPBodyFingerprintMatcher{} + if matcher.Contains("example.ca", "x", fingerprints.FingerprintModeLiteral) { + t.Fatal("expected false for nil client") + } + }) + + t.Run("literal and regex matching", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("Hello from sample app. Error Code: 503")) + })) + t.Cleanup(server.Close) + + domain := strings.TrimPrefix(server.URL, "http://") + matcher := NewHTTPBodyFingerprintMatcher(2 * time.Second) + + if !matcher.Contains(domain, "sample app", fingerprints.FingerprintModeLiteral) { + t.Fatal("expected literal substring match") + } + if !matcher.Contains(domain, `Error Code: \d+`, fingerprints.FingerprintModeRegex) { + t.Fatal("expected regex match") + } + if matcher.Contains(domain, "does-not-exist", fingerprints.FingerprintModeLiteral) { + t.Fatal("did not expect missing literal to match") + } + }) + + t.Run("invalid regex returns false", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("anything")) + })) + t.Cleanup(server.Close) + + domain := strings.TrimPrefix(server.URL, "http://") + matcher := NewHTTPBodyFingerprintMatcher(2 * time.Second) + if matcher.Contains(domain, "(", fingerprints.FingerprintModeRegex) { + t.Fatal("expected false for invalid regex") + } + }) +} diff --git a/scanners/subdomain-takeover/internal/detect/classifier_test.go b/scanners/subdomain-takeover/internal/detect/classifier_test.go new file mode 100644 index 0000000000..e09a055c7d --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/classifier_test.go @@ -0,0 +1,139 @@ +package detect + +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/rs/zerolog" +) + +func TestClassify_ExpectedBehavior(t *testing.T) { + cnameFPs := []fingerprints.CNAMEProviderFingerprint{ + {Name: "Azure", Cname: []string{"azurewebsites.net"}, Nxdomain: true, Fingerprint: "unused"}, + {Name: "Ghost", Cname: []string{"ghost.io"}, Nxdomain: false, Fingerprint: "ghost 404", Mode: fingerprints.FingerprintModeLiteral}, + } + nsFPs := []fingerprints.NSProviderFingerprint{ + {Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, + {Name: "SafeDNS", Status: fingerprints.NSStatusNotVulnerable, HostPatterns: []string{"*.safe-dns.net"}}, + } + + source := fakeSource{cname: cnameFPs, ns: nsFPs} + + t.Run("emits cname and ns findings when both are exploitable", func(t *testing.T) { + rdapMatch := true + input := model.Input{ + DomainKey: "k1", + Results: model.ScanResults{ + Domain: strPtr("a.example.ca"), + CnameRecord: strPtr("a.example.ca. 300 IN CNAME foo.azurewebsites.net."), + RegistrarContext: &model.RegistrarContext{ + LookupSuccess: true, + RegistrarName: "Namecheap", + DelegationMatchesRDAP: &rdapMatch, + }, + NsDelegations: &model.NsDelegations{ + Hosts: []string{"ns1.risky-dns.net"}, + Delegation: model.Delegation{ + LameType: "partial", + }, + }, + }, + } + + findings, err := Classify(input, fakeMatcher{}, source, zerolog.Nop()) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + if len(findings) != 2 { + t.Fatalf("expected 2 findings, got %d", len(findings)) + } + + var sawCNAME, sawNS bool + for _, f := range findings { + if f.RecordType == model.RecordTypeCNAME { + sawCNAME = true + if f.ReasonCode != string(ReasonCNAMEDanglingNXDOMAIN) { + t.Fatalf("unexpected cname reason: %q", f.ReasonCode) + } + } + if f.RecordType == model.RecordTypeNS { + sawNS = true + if f.ReasonCode != string(ReasonNSPartialLameProviderVulnerable) { + t.Fatalf("unexpected ns reason: %q", f.ReasonCode) + } + } + } + if !sawCNAME || !sawNS { + t.Fatalf("missing expected finding types: cname=%v ns=%v", sawCNAME, sawNS) + } + }) + + t.Run("suppresses non-exploitable ns matches", func(t *testing.T) { + input := model.Input{ + DomainKey: "k2", + Results: model.ScanResults{ + Domain: strPtr("b.example.ca"), + CnameRecord: strPtr("b.example.ca. 300 IN CNAME foo.ghost.io."), + NsDelegations: &model.NsDelegations{ + Hosts: []string{"ns1.safe-dns.net"}, + Delegation: model.Delegation{ + LameType: "full", + }, + }, + }, + } + + matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { + return true + }} + + findings, err := Classify(input, matcher, source, zerolog.Nop()) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + + if len(findings) != 1 { + t.Fatalf("expected 1 finding, got %d", len(findings)) + } + if findings[0].RecordType != model.RecordTypeCNAME { + t.Fatalf("expected cname-only finding, got %s", findings[0].RecordType) + } + }) + + t.Run("returns no findings and no panic when evidence absent", func(t *testing.T) { + input := model.Input{DomainKey: "k3", Results: model.ScanResults{}} + findings, err := Classify(input, fakeMatcher{}, source, zerolog.Nop()) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + if len(findings) != 0 { + t.Fatalf("expected no findings, got %d", len(findings)) + } + }) +} + +func TestClassifier_MethodDefaults(t *testing.T) { + classifier := NewClassifier(nil) + if classifier == nil { + t.Fatal("expected classifier") + } + + input := model.Input{DomainKey: "k", Results: model.ScanResults{}} + findings, err := classifier.Classify(input) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + if len(findings) != 0 { + t.Fatalf("expected no findings, got %d", len(findings)) + } + + custom := NewClassifierWithSource(fakeMatcher{}, fakeSource{}) + if custom == nil { + t.Fatal("expected classifier with source") + } + + if custom.WithLogger(zerolog.Nop()) != custom { + t.Fatal("WithLogger should return same classifier pointer") + } +} diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules_test.go b/scanners/subdomain-takeover/internal/detect/cname_rules_test.go new file mode 100644 index 0000000000..a8c2e217c8 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/cname_rules_test.go @@ -0,0 +1,106 @@ +package detect + +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" +) + +func TestMatchCNAMEFingerprints(t *testing.T) { + fps := []fingerprints.CNAMEProviderFingerprint{ + { + Name: "Azure", + Cname: []string{"azurewebsites.net"}, + Nxdomain: true, + Fingerprint: "This web app is stopped", + }, + { + Name: "Ghost", + Cname: []string{"ghost.io"}, + Nxdomain: false, + Fingerprint: "The thing you were looking for is no longer here", + Mode: fingerprints.FingerprintModeLiteral, + }, + } + + t.Run("returns nil when no target match", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "foo.not-a-provider.net", NoResolve: true} + if got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{}); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("nxdomain provider emits dangling reason when unresolved", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "foo.azurewebsites.net", NoResolve: true} + got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{}) + if got == nil { + t.Fatal("expected hit, got nil") + } + if !got.Matched { + t.Fatal("expected matched=true") + } + if got.ReasonCode != ReasonCNAMEDanglingNXDOMAIN { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("nxdomain provider suppressed when resolve evidence exists", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "foo.azurewebsites.net", NoResolve: false} + got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{}) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Matched { + t.Fatal("expected matched=false") + } + if got.ReasonCode != ReasonCNAMETargetMatchMissingNXDOMAIN { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("body fingerprint provider emits when matcher returns true", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "blog.ghost.io", NoResolve: false} + matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { + if domain != "a.example.ca" || mode != fingerprints.FingerprintModeLiteral { + t.Fatalf("unexpected matcher args: domain=%q mode=%q", domain, mode) + } + return true + }} + got := MatchCNAMEFingerprints(evidence, fps, matcher) + if got == nil { + t.Fatal("expected hit, got nil") + } + if !got.Matched { + t.Fatal("expected matched=true") + } + if got.ReasonCode != ReasonCNAMEProviderFingerprintBodyMatch { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("body fingerprint provider suppressed when matcher false", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "blog.ghost.io", NoResolve: false} + got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{containsFn: func(string, string, fingerprints.FingerprintMode) bool { return false }}) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Matched { + t.Fatal("expected matched=false") + } + if got.ReasonCode != ReasonCNAMETargetMatchMissingBodyFP { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) +} + +func TestShouldEmitCNAME(t *testing.T) { + if ShouldEmitCNAME(nil) { + t.Fatal("expected false for nil hit") + } + if ShouldEmitCNAME(&CNAMEHit{Matched: false}) { + t.Fatal("expected false for unmatched hit") + } + if !ShouldEmitCNAME(&CNAMEHit{Matched: true}) { + t.Fatal("expected true for matched hit") + } +} diff --git a/scanners/subdomain-takeover/internal/detect/confidence_test.go b/scanners/subdomain-takeover/internal/detect/confidence_test.go new file mode 100644 index 0000000000..d7efdd2061 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/confidence_test.go @@ -0,0 +1,26 @@ +package detect + +import "testing" + +func TestConfidenceForReason(t *testing.T) { + tests := []struct { + reason ReasonCode + want string + }{ + {ReasonCNAMEDanglingNXDOMAIN, ConfidenceProbable}, + {ReasonCNAMEProviderFingerprintBodyMatch, ConfidenceProbable}, + {ReasonCNAMETargetMatchMissingNXDOMAIN, ConfidenceSuspected}, + {ReasonCNAMETargetMatchMissingBodyFP, ConfidenceSuspected}, + {ReasonNSFullLameProviderVulnerable, ConfidenceConfirmed}, + {ReasonNSPartialLameProviderVulnerable, ConfidenceProbable}, + {ReasonNSLameProviderUnknown, ConfidenceSuspected}, + {ReasonNSProviderMatchOnly, ConfidenceSuspected}, + {ReasonCode("UNKNOWN_REASON"), ConfidenceSuspected}, + } + + for _, tt := range tests { + if got := ConfidenceForReason(tt.reason); got != tt.want { + t.Fatalf("ConfidenceForReason(%q)=%q want=%q", tt.reason, got, tt.want) + } + } +} diff --git a/scanners/subdomain-takeover/internal/detect/evidence_test.go b/scanners/subdomain-takeover/internal/detect/evidence_test.go new file mode 100644 index 0000000000..1dc1006fe9 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/evidence_test.go @@ -0,0 +1,128 @@ +package detect + +import ( + "reflect" + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +func TestExtractCNAMEEvidence(t *testing.T) { + t.Run("returns nil when cname record missing", func(t *testing.T) { + results := minimalResults("app.example.ca") + if got := ExtractCNAMEEvidence(results); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil when domain missing", func(t *testing.T) { + record := "a.example.ca. 300 IN CNAME foo.azurewebsites.net." + results := model.ScanResults{CnameRecord: &record} + if got := ExtractCNAMEEvidence(results); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("extracts normalized target and noresolve true", func(t *testing.T) { + record := "A.Example.CA. 300 IN CNAME Foo.AzureWebsites.NET." + results := minimalResults("app.example.ca") + results.CnameRecord = &record + + got := ExtractCNAMEEvidence(results) + if got == nil { + t.Fatal("expected evidence, got nil") + } + + if got.Domain != "app.example.ca" { + t.Fatalf("unexpected domain: %q", got.Domain) + } + if got.Target != "foo.azurewebsites.net" { + t.Fatalf("unexpected target: %q", got.Target) + } + if !got.NoResolve { + t.Fatal("expected NoResolve=true") + } + }) + + t.Run("sets noresolve false when resolve chain present", func(t *testing.T) { + record := "a.example.ca. 300 IN CNAME foo.azurewebsites.net." + results := minimalResults("app.example.ca") + results.CnameRecord = &record + results.ResolveChain = [][]string{{"a.example.ca CNAME foo.azurewebsites.net"}} + + got := ExtractCNAMEEvidence(results) + if got == nil { + t.Fatal("expected evidence, got nil") + } + if got.NoResolve { + t.Fatal("expected NoResolve=false") + } + }) +} + +func TestExtractNSEvidence(t *testing.T) { + t.Run("returns nil when domain missing", func(t *testing.T) { + input := model.ScanResults{NsDelegations: &model.NsDelegations{Hosts: []string{"ns1.example.net"}}} + if got := ExtractNSEvidence(input); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil when ns delegations missing", func(t *testing.T) { + input := minimalResults("app.example.ca") + if got := ExtractNSEvidence(input); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil when no hosts", func(t *testing.T) { + input := minimalResults("app.example.ca") + input.NsDelegations = &model.NsDelegations{} + if got := ExtractNSEvidence(input); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("normalizes, sorts, and deduplicates hosts", func(t *testing.T) { + input := minimalResults("app.example.ca") + input.NsDelegations = &model.NsDelegations{ + Hosts: []string{"NS2.Example.Net.", "ns1.example.net", "ns2.example.net", ".ns1.example.net."}, + Delegation: model.Delegation{ + LameType: "partial", + }, + } + + got := ExtractNSEvidence(input) + if got == nil { + t.Fatal("expected evidence, got nil") + } + + expectedHosts := []string{"ns1.example.net", "ns2.example.net"} + if !reflect.DeepEqual(got.NSHosts, expectedHosts) { + t.Fatalf("unexpected hosts: got=%v want=%v", got.NSHosts, expectedHosts) + } + if got.Domain != "app.example.ca" { + t.Fatalf("unexpected domain: %q", got.Domain) + } + if got.NSDelegations.Delegation.LameType != "partial" { + t.Fatalf("unexpected lame type: %q", got.NSDelegations.Delegation.LameType) + } + }) +} + +func TestNormalizeHostAndParseHelpers(t *testing.T) { + if got := normalizeHost(".NS1.Example.NET."); got != "ns1.example.net" { + t.Fatalf("normalizeHost mismatch: %q", got) + } + + record := "x.example.ca. 300 IN CNAME target.provider.net." + if got := parseCname(record); got != "target.provider.net" { + t.Fatalf("parseCname mismatch: %q", got) + } + + hosts := []string{"ns3.example.net", "NS1.example.net.", "ns1.example.net", "ns2.example.net"} + expected := []string{"ns1.example.net", "ns2.example.net", "ns3.example.net"} + if got := parseHostnames(hosts); !reflect.DeepEqual(got, expected) { + t.Fatalf("parseHostnames mismatch: got=%v want=%v", got, expected) + } +} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go b/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go new file mode 100644 index 0000000000..34f45f1c3b --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go @@ -0,0 +1,22 @@ +package detect + +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/rs/zerolog" +) + +func TestGlobalFingerprintSource(t *testing.T) { + if err := fingerprints.Load(zerolog.Nop()); err != nil { + t.Fatalf("failed to load fingerprints: %v", err) + } + + src := GlobalFingerprintSource{} + if got := src.CNAME(); len(got) == 0 { + t.Fatal("expected cname source to be populated") + } + if got := src.NS(); len(got) == 0 { + t.Fatal("expected ns source to be populated") + } +} diff --git a/scanners/subdomain-takeover/internal/detect/logger_test.go b/scanners/subdomain-takeover/internal/detect/logger_test.go new file mode 100644 index 0000000000..2487426271 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/logger_test.go @@ -0,0 +1,15 @@ +package detect + +import ( + "io" + "testing" + + "github.com/rs/zerolog" +) + +func TestSetLogger(t *testing.T) { + SetLogger(zerolog.New(io.Discard)) + if got := detectLogger.GetLevel(); got == zerolog.Disabled { + t.Fatalf("expected configured logger, got level=%s", got) + } +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go new file mode 100644 index 0000000000..3b8fccb923 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go @@ -0,0 +1,121 @@ +package detect + +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +func TestGetNSHijackReasonCode(t *testing.T) { + namecheapRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"} + digitalOceanRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."} + invalidRegistrar := &model.RegistrarContext{LookupSuccess: false, RegistrarName: "Namecheap"} + + tests := []struct { + name string + lameType string + provider fingerprints.NSProviderFingerprint + registrar *model.RegistrarContext + want ReasonCode + }{ + { + name: "full vulnerable with registrar mismatch", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSFullLameProviderVulnerable, + }, + { + name: "partial vulnerable with registrar mismatch", + lameType: "partial", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSPartialLameProviderVulnerable, + }, + { + name: "full vulnerable with purchase and mismatch", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerableWithPurchase}, + registrar: namecheapRegistrar, + want: ReasonNSFullLameProviderVulnerable, + }, + { + name: "full non vulnerable", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusNotVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSLameProviderUnknown, + }, + { + name: "unknown lame type", + lameType: "none", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSProviderMatchOnly, + }, + { + name: "same registrar/provider suppressed", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: digitalOceanRegistrar, + want: ReasonNSRegistrarProviderMatch, + }, + { + name: "missing registrar context suppressed", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: nil, + want: ReasonNSRegistrarContextInsufficient, + }, + { + name: "invalid registrar context suppressed", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: invalidRegistrar, + want: ReasonNSRegistrarContextInsufficient, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getNSHijackReasonCode(tt.lameType, tt.provider, tt.registrar) + if got != tt.want { + t.Fatalf("getNSHijackReasonCode()=%q want=%q", got, tt.want) + } + }) + } +} + +func TestNSReasoningHelpers(t *testing.T) { + if !isExploitableProviderStatus(fingerprints.NSStatusVulnerable) { + t.Fatal("expected vulnerable to be exploitable") + } + if !isExploitableProviderStatus(fingerprints.NSStatusVulnerableWithPurchase) { + t.Fatal("expected vulnerable_with_purchase to be exploitable") + } + if isExploitableProviderStatus(fingerprints.NSStatusNotVulnerable) { + t.Fatal("expected not_vulnerable to be non-exploitable") + } + + if got := normalizeLameType(" Partial "); got != "partial" { + t.Fatalf("normalizeLameType mismatch: %q", got) + } + + if !isRegistrarMismatch("Digital Ocean", &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"}) { + t.Fatal("expected registrar mismatch for different providers") + } + if isRegistrarMismatch("Digital Ocean", &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."}) { + t.Fatal("expected same provider to fail mismatch check") + } + + if nsReasonRank(ReasonNSFullLameProviderVulnerable) <= nsReasonRank(ReasonNSPartialLameProviderVulnerable) { + t.Fatal("expected full lame rank > partial lame rank") + } + if nsReasonRank(ReasonNSPartialLameProviderVulnerable) <= nsReasonRank(ReasonNSRegistrarProviderMatch) { + t.Fatal("expected partial lame rank > registrar-provider-match rank") + } + if nsReasonRank(ReasonNSRegistrarProviderMatch) <= nsReasonRank(ReasonNSProviderMatchOnly) { + t.Fatal("expected registrar gate rank > provider-only rank") + } +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go new file mode 100644 index 0000000000..61b41f7e47 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go @@ -0,0 +1,160 @@ +package detect + +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +func TestMatchNSProviderRules(t *testing.T) { + nsFP := []fingerprints.NSProviderFingerprint{ + {Name: "UnknownDNS", Status: fingerprints.NSStatusNotVulnerable, HostPatterns: []string{"*.unknown-dns.net"}}, + {Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, + } + + namecheapRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"} + digitalOceanRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."} + + t.Run("returns nil for missing hosts", func(t *testing.T) { + evidence := NSEvidence{Domain: "a.example.ca"} + if got := MatchNSProviderRules(evidence, nsFP); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil for missing fingerprints", func(t *testing.T) { + evidence := NSEvidence{Domain: "a.example.ca", NSHosts: []string{"ns1.risky-dns.net"}} + if got := MatchNSProviderRules(evidence, nil); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil when no provider matches", func(t *testing.T) { + evidence := NSEvidence{Domain: "a.example.ca", NSHosts: []string{"ns1.nomatch.net"}} + if got := MatchNSProviderRules(evidence, nsFP); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("matches vulnerable full lame and emits correct reason", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.risky-dns.net"}, + Registrar: namecheapRegistrar, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + got := MatchNSProviderRules(evidence, nsFP) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Provider != "Digital Ocean" { + t.Fatalf("unexpected provider: %q", got.Provider) + } + if got.ReasonCode != ReasonNSFullLameProviderVulnerable { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if !got.Matched { + t.Fatal("expected matched=true") + } + }) + + t.Run("ranks vulnerable higher than unknown", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.unknown-dns.net", "ns1.risky-dns.net"}, + Registrar: namecheapRegistrar, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "partial", + }}, + } + got := MatchNSProviderRules(evidence, nsFP) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Provider != "Digital Ocean" { + t.Fatalf("expected risky provider to win ranking, got %q", got.Provider) + } + if got.ReasonCode != ReasonNSPartialLameProviderVulnerable { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("non-vulnerable full lame is classified but not emittable", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.unknown-dns.net"}, + Registrar: namecheapRegistrar, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + got := MatchNSProviderRules(evidence, nsFP) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.ReasonCode != ReasonNSLameProviderUnknown { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if got.Matched { + t.Fatal("expected matched=false for non-vulnerable state") + } + }) + + t.Run("suppresses vulnerable provider when registrar matches provider", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.risky-dns.net"}, + Registrar: digitalOceanRegistrar, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + + got := MatchNSProviderRules(evidence, nsFP) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.ReasonCode != ReasonNSRegistrarProviderMatch { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if got.Matched { + t.Fatal("expected matched=false when registrar and provider match") + } + }) + + t.Run("suppresses vulnerable provider when registrar context missing", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.risky-dns.net"}, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + + got := MatchNSProviderRules(evidence, nsFP) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.ReasonCode != ReasonNSRegistrarContextInsufficient { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if got.Matched { + t.Fatal("expected matched=false when registrar context is missing") + } + }) +} + +func TestShouldEmitNSHijack(t *testing.T) { + if ShouldEmitNSHijack(nil) { + t.Fatal("expected false for nil hit") + } + if ShouldEmitNSHijack(&NSHit{Matched: false}) { + t.Fatal("expected false for unmatched hit") + } + if !ShouldEmitNSHijack(&NSHit{Matched: true}) { + t.Fatal("expected true for matched hit") + } +} diff --git a/scanners/subdomain-takeover/internal/detect/test_helpers_test.go b/scanners/subdomain-takeover/internal/detect/test_helpers_test.go new file mode 100644 index 0000000000..057f7101eb --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/test_helpers_test.go @@ -0,0 +1,33 @@ +package detect + +import ( + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +type fakeMatcher struct { + containsFn func(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool +} + +func (f fakeMatcher) Contains(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { + if f.containsFn == nil { + return false + } + return f.containsFn(domain, fingerprint, mode) +} + +type fakeSource struct { + cname []fingerprints.CNAMEProviderFingerprint + ns []fingerprints.NSProviderFingerprint +} + +func (f fakeSource) CNAME() []fingerprints.CNAMEProviderFingerprint { return f.cname } +func (f fakeSource) NS() []fingerprints.NSProviderFingerprint { return f.ns } + +func strPtr(v string) *string { return &v } + +func boolPtr(v bool) *bool { return &v } + +func minimalResults(domain string) model.ScanResults { + return model.ScanResults{Domain: strPtr(domain)} +} diff --git a/scanners/subdomain-takeover/internal/fingerprints/loader_test.go b/scanners/subdomain-takeover/internal/fingerprints/loader_test.go new file mode 100644 index 0000000000..f28d2a0d1c --- /dev/null +++ b/scanners/subdomain-takeover/internal/fingerprints/loader_test.go @@ -0,0 +1,94 @@ +package fingerprints + +import ( + "sync" + "testing" + + "github.com/rs/zerolog" +) + +func TestLoad(t *testing.T) { + origErr := loadErr + origCNAME := cnameProviderFingerprints + origNS := nsProviderFingerprints + + loadOnce = sync.Once{} + loadErr = nil + cnameProviderFingerprints = nil + nsProviderFingerprints = nil + + t.Cleanup(func() { + loadOnce = sync.Once{} + loadErr = origErr + cnameProviderFingerprints = origCNAME + nsProviderFingerprints = origNS + }) + + err := Load(zerolog.Nop()) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if len(CNAME()) == 0 { + t.Fatal("expected cname fingerprints to be loaded") + } + if len(NS()) == 0 { + t.Fatal("expected ns fingerprints to be loaded") + } +} + +func TestValidateFingerprintHelpers(t *testing.T) { + if err := validateCNAMEFingerprints([]CNAMEProviderFingerprint{{Name: "", Cname: []string{"x"}, Fingerprint: "fp"}}); err == nil { + t.Fatal("expected error for missing cname fingerprint name") + } + if err := validateCNAMEFingerprints([]CNAMEProviderFingerprint{{Name: "ok", Cname: nil, Fingerprint: "fp"}}); err == nil { + t.Fatal("expected error for missing cname patterns") + } + if err := validateCNAMEFingerprints([]CNAMEProviderFingerprint{{Name: "ok", Cname: []string{""}, Fingerprint: "fp"}}); err == nil { + t.Fatal("expected error for empty cname pattern") + } + if err := validateCNAMEFingerprints([]CNAMEProviderFingerprint{{Name: "ok", Cname: []string{"example.net"}, Fingerprint: ""}}); err == nil { + t.Fatal("expected error for missing fingerprint") + } + + validCNAME := []CNAMEProviderFingerprint{{ + Name: "ok", + Cname: []string{"example.net"}, + Fingerprint: "service unavailable", + }} + if err := validateCNAMEFingerprints(validCNAME); err != nil { + t.Fatalf("expected valid cname fingerprints, got error: %v", err) + } + if validCNAME[0].Mode != FingerprintModeLiteral { + t.Fatalf("expected normalized literal mode, got %q", validCNAME[0].Mode) + } + + if err := validateNSFingerprints([]NSProviderFingerprint{{Name: "", HostPatterns: []string{"*.example.net"}}}); err == nil { + t.Fatal("expected error for missing ns name") + } + if err := validateNSFingerprints([]NSProviderFingerprint{{Name: "ok", HostPatterns: nil}}); err == nil { + t.Fatal("expected error for missing ns host patterns") + } + if err := validateNSFingerprints([]NSProviderFingerprint{{Name: "ok", HostPatterns: []string{""}}}); err == nil { + t.Fatal("expected error for empty ns host pattern") + } + if err := validateNSFingerprints([]NSProviderFingerprint{{Name: "ok", HostPatterns: []string{"*.example.net"}}}); err != nil { + t.Fatalf("expected valid ns fingerprints, got error: %v", err) + } +} + +func TestLoadJSON(t *testing.T) { + t.Run("returns read error for missing file", func(t *testing.T) { + var out []CNAMEProviderFingerprint + err := loadJSON("data/missing.json", &out) + if err == nil { + t.Fatal("expected read error") + } + }) + + t.Run("returns decode error for invalid target", func(t *testing.T) { + err := loadJSON("data/cname_fingerprints.json", nil) + if err == nil { + t.Fatal("expected decode error") + } + }) +} diff --git a/scanners/subdomain-takeover/internal/fingerprints/types_test.go b/scanners/subdomain-takeover/internal/fingerprints/types_test.go new file mode 100644 index 0000000000..96be0d1183 --- /dev/null +++ b/scanners/subdomain-takeover/internal/fingerprints/types_test.go @@ -0,0 +1,47 @@ +package fingerprints + +import "testing" + +func TestContainsNSHostAndWildcardMatch(t *testing.T) { + fp := NSProviderFingerprint{Name: "Test", HostPatterns: []string{"*.example.net", "ns.fixed.io"}} + + if !fp.ContainsNSHost("NS1.EXAMPLE.NET.") { + t.Fatal("expected wildcard match for ns1.example.net") + } + if !fp.ContainsNSHost("ns.fixed.io") { + t.Fatal("expected exact match for ns.fixed.io") + } + if fp.ContainsNSHost("other.net") { + t.Fatal("did not expect match for other.net") + } + + if !wildcardHostMatch("abc*xyz*io", "abc-123-xyz-final-io") { + t.Fatal("expected multi-star wildcard to match") + } + if wildcardHostMatch("abc*xyz", "zabc123xyz") { + t.Fatal("did not expect prefix mismatch to match") + } +} + +func TestContainsTargetAndNormalizeMode(t *testing.T) { + fp := CNAMEProviderFingerprint{Cname: []string{"azurewebsites.net"}} + if !fp.ContainsTarget("foo.azurewebsites.net") { + t.Fatal("expected suffix match") + } + if fp.ContainsTarget("foo.example.net") { + t.Fatal("did not expect suffix mismatch") + } + + if got := NormalizeMode(FingerprintModeLiteral, "foo"); got != FingerprintModeLiteral { + t.Fatalf("unexpected mode: %q", got) + } + if got := NormalizeMode("", "service unavailable"); got != FingerprintModeLiteral { + t.Fatalf("unexpected inferred mode: %q", got) + } + if got := NormalizeMode("", "Error: .* not found"); got != FingerprintModeRegex { + t.Fatalf("unexpected inferred mode: %q", got) + } + if got := NormalizeMode("", "foo\\d+"); got != FingerprintModeRegex { + t.Fatalf("unexpected inferred mode: %q", got) + } +} diff --git a/scanners/subdomain-takeover/internal/messaging/health_test.go b/scanners/subdomain-takeover/internal/messaging/health_test.go new file mode 100644 index 0000000000..06829d6757 --- /dev/null +++ b/scanners/subdomain-takeover/internal/messaging/health_test.go @@ -0,0 +1,52 @@ +package messaging + +import ( + "testing" + + "github.com/nats-io/nats.go" +) + +type fakeConnState struct { + closed bool + connected bool + status nats.Status +} + +func (f fakeConnState) IsClosed() bool { return f.closed } +func (f fakeConnState) IsConnected() bool { return f.connected } +func (f fakeConnState) Status() nats.Status { + return f.status +} + +func TestCheckConnection(t *testing.T) { + err := CheckConnection(nil) + if err == nil { + t.Fatal("expected error for nil connection") + } + if err.Error() != "nats connection is nil" { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestCheckConnectionState(t *testing.T) { + t.Run("closed connection returns closed error", func(t *testing.T) { + err := checkConnectionState(fakeConnState{closed: true}) + if err == nil || err.Error() != "nats connection is closed" { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("not connected returns status error", func(t *testing.T) { + err := checkConnectionState(fakeConnState{connected: false, status: nats.DISCONNECTED}) + if err == nil || err.Error() != "nats not connected (status=DISCONNECTED)" { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("connected state returns nil", func(t *testing.T) { + err := checkConnectionState(fakeConnState{connected: true, status: nats.CONNECTED}) + if err != nil { + t.Fatalf("expected nil error, got: %v", err) + } + }) +} diff --git a/scanners/subdomain-takeover/internal/messaging/publisher_test.go b/scanners/subdomain-takeover/internal/messaging/publisher_test.go new file mode 100644 index 0000000000..5e7f83b835 --- /dev/null +++ b/scanners/subdomain-takeover/internal/messaging/publisher_test.go @@ -0,0 +1,113 @@ +package messaging + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type fakePublishClient struct { + err error + ack *jetstream.PubAck + called int + lastSubj string + lastBytes []byte +} + +func (f *fakePublishClient) Publish(_ context.Context, subj string, data []byte, _ ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + f.called++ + f.lastSubj = subj + f.lastBytes = append([]byte(nil), data...) + if f.err != nil { + return nil, f.err + } + if f.ack != nil { + return f.ack, nil + } + return &jetstream.PubAck{}, nil +} + +func TestNewPublisher(t *testing.T) { + client := &fakePublishClient{} + p := NewPublisher(zerolog.Nop(), client, "scans.findings.subdomain_takeover") + if p == nil { + t.Fatal("expected publisher instance") + } + if p.subject != "scans.findings.subdomain_takeover" { + t.Fatalf("unexpected subject: %q", p.subject) + } +} + +func TestPublisherPublish(t *testing.T) { + origMarshal := marshalFinding + t.Cleanup(func() { marshalFinding = origMarshal }) + + finding := model.Finding{ + Domain: "a.example.ca", + DomainKey: "123", + RecordType: model.RecordTypeNS, + Target: "ns1.risky-dns.net", + Provider: "RiskyDNS", + Confidence: "probable", + ReasonCode: "NS_PARTIAL_LAME_PROVIDER_VULNERABLE", + } + + t.Run("publishes serialized finding", func(t *testing.T) { + client := &fakePublishClient{} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain_takeover"} + + err := p.Publish(context.Background(), finding) + if err != nil { + t.Fatalf("unexpected publish error: %v", err) + } + if client.called != 1 { + t.Fatalf("expected one publish call, got %d", client.called) + } + if client.lastSubj != "scans.findings.subdomain_takeover" { + t.Fatalf("unexpected subject: %q", client.lastSubj) + } + + var got model.Finding + if err := json.Unmarshal(client.lastBytes, &got); err != nil { + t.Fatalf("payload not valid json: %v", err) + } + if got != finding { + t.Fatalf("unexpected payload: got=%+v want=%+v", got, finding) + } + }) + + t.Run("returns publish client error", func(t *testing.T) { + client := &fakePublishClient{err: errors.New("publish failed")} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain_takeover"} + + err := p.Publish(context.Background(), finding) + if err == nil { + t.Fatal("expected publish error") + } + if client.called != 1 { + t.Fatalf("expected one publish call, got %d", client.called) + } + }) + + t.Run("returns marshal error and does not publish", func(t *testing.T) { + marshalFinding = func(v any) ([]byte, error) { + return nil, errors.New("marshal failed") + } + + client := &fakePublishClient{} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain_takeover"} + + err := p.Publish(context.Background(), finding) + if err == nil { + t.Fatal("expected marshal error") + } + if client.called != 0 { + t.Fatalf("expected no publish call, got %d", client.called) + } + }) +} From 9af65e62c946b17da308d8b36d93ee39fdf02b74 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Tue, 4 Aug 2026 14:19:59 -0300 Subject: [PATCH 50/65] subdomain-takeover: add docs and build/deploy config --- scanners/subdomain-takeover/README.md | 155 ++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 scanners/subdomain-takeover/README.md diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md new file mode 100644 index 0000000000..0677e91e3c --- /dev/null +++ b/scanners/subdomain-takeover/README.md @@ -0,0 +1,155 @@ +# subdomain-takeover-service + +Go microservice that consumes DNS scan results and emits normalized subdomain takeover findings. + +## What it does + +1. Consumes from `scans.dns_scanner_results` (JetStream stream: `SCANS`) +2. Evaluates passive CNAME and NS takeover risk using provider fingerprints +3. Assigns confidence (`suspected`, `probable`, `confirmed`) +4. Publishes findings to `scans.findings.subdomain_takeover` + +## Current detection scope + +Implemented today: + +- CNAME passive detection +- NS passive detection (vulnerable-only emission policy) +- Deterministic NS candidate selection when multiple providers/hosts match +- Nil-safe evidence extraction for CNAME and NS +- Classifier refactor to support injected fingerprint sources (`FingerprintSource`) + +Current NS behavior: + +- NS provider/host matches are evaluated and ranked +- Findings are emitted only for exploitable NS outcomes: + - `NS_FULL_LAME_PROVIDER_VULNERABLE` + - `NS_PARTIAL_LAME_PROVIDER_VULNERABLE` +- Non-exploitable NS matches (`NS_LAME_PROVIDER_UNKNOWN`, `NS_PROVIDER_MATCH_ONLY`) are classified but not emitted + +## Passive-only detection policy + +This service uses passive checks only. +It does not attempt account takeover, resource claiming, or any active validation. + +As a result, confidence is conservative and evidence-driven: + +- `suspected`: weak or incomplete takeover indicators +- `probable`: strong passive indicators with known exploitable conditions +- `confirmed`: reserved for deterministic passive signatures only + +## Confidence model (current) + +### CNAME + +- `suspected` + - provider target matched, but required passive signal is missing +- `probable` + - dangling target with NXDOMAIN where required, or + - provider fingerprint body match for providers that require body verification +- `confirmed` + - reserved; not currently emitted by CNAME rules + +### NS + +- `confirmed` + - full lame delegation + exploitable provider status +- `probable` + - partial lame delegation + exploitable provider status +- `suspected` + - reason codes exist for weaker NS states, but these are currently not emitted + +## Internal package layout + +Detection logic was split into focused files to reduce coupling: + +- `internal/detect/cname_rules.go` +- `internal/detect/ns_rules.go` +- `internal/detect/ns_reasoning.go` +- `internal/detect/cname_evidence.go` +- `internal/detect/ns_evidence.go` +- `internal/detect/fingerprint_source.go` + +`Classifier` now supports dependency injection via `NewClassifierWithSource(...)`, while `NewClassifier(...)` defaults to global loaded fingerprints. + +## Fingerprint data + +Provider fingerprint data is stored in: + +- `internal/fingerprints/data/cname_fingerprints.json` +- `internal/fingerprints/data/ns_fingerprints.json` + +The service embeds and validates these files at startup (`fingerprints.Load`). + +For CNAME body matching: + +- `mode: literal` performs plain substring search +- `mode: regex` performs regular expression matching +- if `mode` is omitted, the loader applies backward-compatible inference + +## Prerequisites + +- Go 1.25+ +- NATS with JetStream enabled +- `nats` CLI (optional, for local testing) + +## Quick start + +```bash +go mod tidy +go run ./cmd/service +``` + +## Local commands + +This service includes a local `Makefile` for common workflows: + +```bash +make help # list targets +make run # run the service +make test # run tests +make lint # fmt-check + vet +make build # build bin/subdomain-takeover +make ci # lint + test + build +``` + +Environment variables: + +- `NATS_URL` (default: `nats://localhost:4222`) +- `NATS_STREAM` (default: `SCANS`) +- `SUBJECT_IN` (default: `scans.dns_scanner_results`) +- `SUBJECT_OUT` (default: `scans.findings.subdomain_takeover`) +- `DURABLE_NAME` (default: `subdomain_takeover`) +- `WORKER_COUNT` (default: `5`) +- `LOG_LEVEL` (default: `info`) + +## Message ack behavior + +The worker uses explicit JetStream ack semantics: + +- decode failures -> `Term()` (drop poison message) +- classify/publish failures -> `Nak()` (retryable) +- successful processing -> `Ack()` + +## Local test flow + +Publish a test request: + +```bash +nats pub scans.dns_scanner_results '{"domain_key":"12345","results":{"domain":"example.gc.ca","resolve_chain":[["old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net."]],"cname_record":"old-app.example.gc.ca. 300 IN CNAME old-app.azurewebsites.net.","ns_delegations":{"ns_hosts":["ns1.example-dns-provider.net"],"ns_checks":[],"ns_delegation":{"total_ns":1,"authoritative_ok":0,"lame_count":1,"lame_type":"full"},"error":""}}}' +``` + +Watch findings: + +```bash +nats sub "scans.findings.subdomain_takeover" +``` + +## Next steps / nice-to-haves + +1. Add table-driven tests for NS reason mapping, ranking, and emission gating. +2. Add classifier integration tests using injected `FingerprintSource`. +3. Document and implement explicit policy for `edge_case` and `registration_closed` NS provider statuses. +4. Add structured debug logs for NS matching decisions (host, provider, status, reason). +5. Add registrar-context mismatch handling (`RegistrarMismatch`) and confidence policy. +6. Add metrics (counts by reason code, emitted vs suppressed) for production observability. From 949098298c068d13ea1d07791bf3dc1f589d2e9e Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 14 Aug 2026 13:21:47 -0300 Subject: [PATCH 51/65] update emitted finding payload structure for generic findings processor --- scanners/subdomain-takeover/README.md | 35 +++++- .../internal/config/config.go | 2 +- .../internal/messaging/publisher.go | 25 ++++- .../internal/messaging/publisher_test.go | 102 +++++++++++++++--- .../internal/model/finding.go | 65 +++++++++++ 5 files changed, 206 insertions(+), 23 deletions(-) diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md index 0677e91e3c..947c91357f 100644 --- a/scanners/subdomain-takeover/README.md +++ b/scanners/subdomain-takeover/README.md @@ -7,7 +7,34 @@ Go microservice that consumes DNS scan results and emits normalized subdomain ta 1. Consumes from `scans.dns_scanner_results` (JetStream stream: `SCANS`) 2. Evaluates passive CNAME and NS takeover risk using provider fingerprints 3. Assigns confidence (`suspected`, `probable`, `confirmed`) -4. Publishes findings to `scans.findings.subdomain_takeover` +4. Publishes findings to `scans.findings.subdomain-takeover` + +Published finding event contract: + +```json +{ + "source": "subdomain-takeover", + "findingType": "subdomain-takeover-cname", + "domainKey": "12345", + "subject": "old-app.example.gc.ca", + "confidence": "probable", + "reasonCode": "CNAME_DANGLING_NXDOMAIN", + "observedAt": "2026-08-07T12:34:56Z", + "evidence": { + "target": "old-app.azurewebsites.net", + "recordType": "CNAME" + }, + "attributes": { + "provider": "azure", + "lameType": "" + } +} +``` + +`findingType` values emitted by this service: + +- `subdomain-takeover-cname` +- `subdomain-takeover-ns` ## Current detection scope @@ -118,8 +145,8 @@ Environment variables: - `NATS_URL` (default: `nats://localhost:4222`) - `NATS_STREAM` (default: `SCANS`) - `SUBJECT_IN` (default: `scans.dns_scanner_results`) -- `SUBJECT_OUT` (default: `scans.findings.subdomain_takeover`) -- `DURABLE_NAME` (default: `subdomain_takeover`) +- `SUBJECT_OUT` (default: `scans.findings.subdomain-takeover`) +- `DURABLE_NAME` (default: `subdomain-takeover`) - `WORKER_COUNT` (default: `5`) - `LOG_LEVEL` (default: `info`) @@ -142,7 +169,7 @@ nats pub scans.dns_scanner_results '{"domain_key":"12345","results":{"domain":"e Watch findings: ```bash -nats sub "scans.findings.subdomain_takeover" +nats sub "scans.findings.subdomain-takeover" ``` ## Next steps / nice-to-haves diff --git a/scanners/subdomain-takeover/internal/config/config.go b/scanners/subdomain-takeover/internal/config/config.go index 053993d380..56dda2e7cb 100644 --- a/scanners/subdomain-takeover/internal/config/config.go +++ b/scanners/subdomain-takeover/internal/config/config.go @@ -9,7 +9,7 @@ type Config struct { NATSURL string `envconfig:"NATS_URL" default:"nats://localhost:4222"` NATSStream string `envconfig:"NATS_STREAM" default:"SCANS"` SubjectIn string `envconfig:"SUBJECT_IN" default:"scans.dns_scanner_results"` - SubjectOut string `envconfig:"SUBJECT_OUT" default:"scans.findings.subdomain_takeover"` + SubjectOut string `envconfig:"SUBJECT_OUT" default:"scans.findings.subdomain-takeover"` DurableName string `envconfig:"DURABLE_NAME" default:"subdomain_takeover"` WorkerCount int `envconfig:"WORKER_COUNT" default:"5"` LogLevel zerolog.Level `envconfig:"LOG_LEVEL" default:"info"` diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go index c903f05e00..9083980cf9 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -3,6 +3,8 @@ package messaging import ( "context" "encoding/json" + "fmt" + "time" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" "github.com/nats-io/nats.go/jetstream" @@ -10,6 +12,7 @@ import ( ) var marshalFinding = json.Marshal +var nowUTC = func() time.Time { return time.Now().UTC() } type Publisher struct { logger zerolog.Logger @@ -26,7 +29,18 @@ func NewPublisher(logger zerolog.Logger, js publishClient, subject string) *Publ } func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { - payload, err := marshalFinding(finding) + event, err := model.NewFindingEventFromFinding(finding, nowUTC()) + if err != nil { + p.logger.Error(). + Err(err). + Str("domain", finding.Domain). + Str("domain_key", finding.DomainKey). + Str("record_type", string(finding.RecordType)). + Msg("finding event mapping failed") + return err + } + + payload, err := marshalFinding(event) if err != nil { p.logger.Error(). Err(err). @@ -35,7 +49,8 @@ func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { Str("record_type", string(finding.RecordType)). Str("reason_code", finding.ReasonCode). Str("confidence", finding.Confidence). - Msg("marshal finding failed") + Str("finding_type", event.FindingType). + Msg("marshal finding event failed") return err } @@ -47,9 +62,10 @@ func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { Str("record_type", string(finding.RecordType)). Str("reason_code", finding.ReasonCode). Str("confidence", finding.Confidence). + Str("finding_type", event.FindingType). Str("subject", p.subject). Msg("publish failed") - return err + return fmt.Errorf("publish finding event: %w", err) } p.logger.Debug(). @@ -58,7 +74,8 @@ func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { Str("record_type", string(finding.RecordType)). Str("reason_code", finding.ReasonCode). Str("confidence", finding.Confidence). + Str("finding_type", event.FindingType). Str("subject", p.subject). - Msg("finding published") + Msg("finding event published") return nil } diff --git a/scanners/subdomain-takeover/internal/messaging/publisher_test.go b/scanners/subdomain-takeover/internal/messaging/publisher_test.go index 5e7f83b835..ca19ea62d2 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher_test.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "testing" + "time" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" "github.com/nats-io/nats.go/jetstream" @@ -34,20 +35,25 @@ func (f *fakePublishClient) Publish(_ context.Context, subj string, data []byte, func TestNewPublisher(t *testing.T) { client := &fakePublishClient{} - p := NewPublisher(zerolog.Nop(), client, "scans.findings.subdomain_takeover") + p := NewPublisher(zerolog.Nop(), client, "scans.findings.subdomain-takeover") if p == nil { t.Fatal("expected publisher instance") } - if p.subject != "scans.findings.subdomain_takeover" { + if p.subject != "scans.findings.subdomain-takeover" { t.Fatalf("unexpected subject: %q", p.subject) } } func TestPublisherPublish(t *testing.T) { origMarshal := marshalFinding + origNowUTC := nowUTC t.Cleanup(func() { marshalFinding = origMarshal }) + t.Cleanup(func() { nowUTC = origNowUTC }) - finding := model.Finding{ + fixedNow := time.Date(2026, 8, 7, 12, 34, 56, 0, time.UTC) + nowUTC = func() time.Time { return fixedNow } + + nsFinding := model.Finding{ Domain: "a.example.ca", DomainKey: "123", RecordType: model.RecordTypeNS, @@ -56,36 +62,88 @@ func TestPublisherPublish(t *testing.T) { Confidence: "probable", ReasonCode: "NS_PARTIAL_LAME_PROVIDER_VULNERABLE", } + cnameFinding := model.Finding{ + Domain: "b.example.ca", + DomainKey: "456", + RecordType: model.RecordTypeCNAME, + Target: "old-app.azurewebsites.net", + Provider: "azure", + Confidence: "probable", + ReasonCode: "CNAME_DANGLING_NXDOMAIN", + } - t.Run("publishes serialized finding", func(t *testing.T) { + t.Run("publishes ns finding event payload", func(t *testing.T) { client := &fakePublishClient{} - p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain_takeover"} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain-takeover"} - err := p.Publish(context.Background(), finding) + err := p.Publish(context.Background(), nsFinding) if err != nil { t.Fatalf("unexpected publish error: %v", err) } if client.called != 1 { t.Fatalf("expected one publish call, got %d", client.called) } - if client.lastSubj != "scans.findings.subdomain_takeover" { + if client.lastSubj != "scans.findings.subdomain-takeover" { t.Fatalf("unexpected subject: %q", client.lastSubj) } - var got model.Finding + var got model.FindingEvent + if err := json.Unmarshal(client.lastBytes, &got); err != nil { + t.Fatalf("payload not valid json: %v", err) + } + + if got.Source != "subdomain-takeover" { + t.Fatalf("unexpected source: %q", got.Source) + } + if got.FindingType != "subdomain-takeover-ns" { + t.Fatalf("unexpected findingType: %q", got.FindingType) + } + if got.DomainKey != nsFinding.DomainKey { + t.Fatalf("unexpected domainKey: %q", got.DomainKey) + } + if got.Subject != nsFinding.Domain { + t.Fatalf("unexpected subject: %q", got.Subject) + } + if got.ObservedAt != fixedNow.Format(time.RFC3339) { + t.Fatalf("unexpected observedAt: %q", got.ObservedAt) + } + if got.Evidence["target"] != nsFinding.Target { + t.Fatalf("unexpected evidence target: %v", got.Evidence["target"]) + } + if got.Evidence["recordType"] != string(nsFinding.RecordType) { + t.Fatalf("unexpected evidence recordType: %v", got.Evidence["recordType"]) + } + if got.Attributes["provider"] != nsFinding.Provider { + t.Fatalf("unexpected attributes provider: %v", got.Attributes["provider"]) + } + if got.Attributes["lameType"] != nsFinding.LameType { + t.Fatalf("unexpected attributes lameType: %v", got.Attributes["lameType"]) + } + }) + + t.Run("maps cname record to cname findingType", func(t *testing.T) { + client := &fakePublishClient{} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain-takeover"} + + err := p.Publish(context.Background(), cnameFinding) + if err != nil { + t.Fatalf("unexpected publish error: %v", err) + } + + var got model.FindingEvent if err := json.Unmarshal(client.lastBytes, &got); err != nil { t.Fatalf("payload not valid json: %v", err) } - if got != finding { - t.Fatalf("unexpected payload: got=%+v want=%+v", got, finding) + if got.FindingType != "subdomain-takeover-cname" { + t.Fatalf("unexpected findingType: %q", got.FindingType) } }) t.Run("returns publish client error", func(t *testing.T) { client := &fakePublishClient{err: errors.New("publish failed")} - p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain_takeover"} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain-takeover"} - err := p.Publish(context.Background(), finding) + err := p.Publish(context.Background(), nsFinding) if err == nil { t.Fatal("expected publish error") } @@ -100,9 +158,9 @@ func TestPublisherPublish(t *testing.T) { } client := &fakePublishClient{} - p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain_takeover"} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain-takeover"} - err := p.Publish(context.Background(), finding) + err := p.Publish(context.Background(), nsFinding) if err == nil { t.Fatal("expected marshal error") } @@ -110,4 +168,20 @@ func TestPublisherPublish(t *testing.T) { t.Fatalf("expected no publish call, got %d", client.called) } }) + + t.Run("returns mapping error for unsupported record type", func(t *testing.T) { + client := &fakePublishClient{} + p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain-takeover"} + + badFinding := nsFinding + badFinding.RecordType = model.RecordType("TXT") + + err := p.Publish(context.Background(), badFinding) + if err == nil { + t.Fatal("expected mapping error") + } + if client.called != 0 { + t.Fatalf("expected no publish call, got %d", client.called) + } + }) } diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go index 66e98a2f35..7563a658f9 100644 --- a/scanners/subdomain-takeover/internal/model/finding.go +++ b/scanners/subdomain-takeover/internal/model/finding.go @@ -1,5 +1,14 @@ package model +import "time" + +type FindingType string + +const ( + FindingTypeSubdomainTakeoverCNAME FindingType = "subdomain-takeover-cname" + FindingTypeSubdomainTakeoverNS FindingType = "subdomain-takeover-ns" +) + type RecordType string const ( @@ -17,3 +26,59 @@ type Finding struct { Confidence string `json:"confidence"` ReasonCode string `json:"reason_code"` } + +type FindingEvent struct { + Source string `json:"source"` + FindingType string `json:"findingType"` + DomainKey string `json:"domainKey"` + Subject string `json:"subject"` + Confidence string `json:"confidence"` + ReasonCode string `json:"reasonCode,omitempty"` + ObservedAt string `json:"observedAt"` + Evidence map[string]any `json:"evidence,omitempty"` + Attributes map[string]any `json:"attributes,omitempty"` +} + +func NewFindingEventFromFinding(f Finding, now time.Time) (FindingEvent, error) { + findingType, err := FindingTypeForRecord(f.RecordType) + if err != nil { + return FindingEvent{}, err + } + + return FindingEvent{ + Source: "subdomain-takeover", + FindingType: findingType, + DomainKey: f.DomainKey, + Subject: f.Domain, + Confidence: f.Confidence, + ReasonCode: f.ReasonCode, + ObservedAt: now.UTC().Format(time.RFC3339), + Evidence: map[string]any{ + "target": f.Target, + "recordType": f.RecordType, + }, + Attributes: map[string]any{ + "provider": f.Provider, + "lameType": f.LameType, + }, + }, nil +} + +func FindingTypeForRecord(recordType RecordType) (string, error) { + switch recordType { + case RecordTypeCNAME: + return string(FindingTypeSubdomainTakeoverCNAME), nil + case RecordTypeNS: + return string(FindingTypeSubdomainTakeoverNS), nil + default: + return "", ErrUnsupportedRecordType + } +} + +var ErrUnsupportedRecordType = unsupportedRecordTypeError{} + +type unsupportedRecordTypeError struct{} + +func (unsupportedRecordTypeError) Error() string { + return "unsupported record type" +} From 1d16dbf7eee815273b529760f828d290c726539f Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Thu, 27 Aug 2026 16:00:38 -0300 Subject: [PATCH 52/65] improve cname target pattern matching --- scanners/subdomain-takeover/internal/fingerprints/types.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scanners/subdomain-takeover/internal/fingerprints/types.go b/scanners/subdomain-takeover/internal/fingerprints/types.go index ac275d8430..28da886e6e 100644 --- a/scanners/subdomain-takeover/internal/fingerprints/types.go +++ b/scanners/subdomain-takeover/internal/fingerprints/types.go @@ -84,14 +84,15 @@ func wildcardHostMatch(pattern, host string) bool { } func (f *CNAMEProviderFingerprint) ContainsTarget(target string) bool { + target = strings.ToLower(strings.TrimSuffix(strings.TrimSpace(target), ".")) for _, cname := range f.Cname { - if strings.HasSuffix(target, cname) { + pattern := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(cname), ".")) + if target == pattern || strings.HasSuffix(target, "."+pattern) { return true } } return false } - func NormalizeMode(mode FingerprintMode, fingerprint string) FingerprintMode { if mode == FingerprintModeLiteral || mode == FingerprintModeRegex { return mode From ae20f5dcc181656f931f579b351b7281e17ac1a4 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Thu, 27 Aug 2026 16:01:03 -0300 Subject: [PATCH 53/65] improve nats bootstrap configs on service restart --- .../subdomain-takeover/internal/bootstrap/nats.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scanners/subdomain-takeover/internal/bootstrap/nats.go b/scanners/subdomain-takeover/internal/bootstrap/nats.go index ea2fc49e21..3c9bd40703 100644 --- a/scanners/subdomain-takeover/internal/bootstrap/nats.go +++ b/scanners/subdomain-takeover/internal/bootstrap/nats.go @@ -2,6 +2,7 @@ package bootstrap import ( "context" + "fmt" "time" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" @@ -30,20 +31,18 @@ func NewRuntimeDeps(ctx context.Context, cfg *config.Config, logger zerolog.Logg return nil, err } - logger.Debug().Str("stream", cfg.NATSStream).Strs("subjects", []string{cfg.SubjectIn, cfg.SubjectOut}).Msg("creating or updating stream") - stream, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ - Name: cfg.NATSStream, - Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, - }) + logger.Debug().Str("stream", cfg.NATSStream).Msg("loading existing stream") + stream, err := js.Stream(ctx, cfg.NATSStream) if err != nil { nc.Close() - return nil, err + return nil, fmt.Errorf("stream %q not found or inaccessible: %w", cfg.NATSStream, err) } logger.Debug().Str("durable", cfg.DurableName).Msg("creating or updating consumer") cons, err := stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ - Durable: cfg.DurableName, - AckPolicy: jetstream.AckExplicitPolicy, + Durable: cfg.DurableName, + AckPolicy: jetstream.AckExplicitPolicy, + FilterSubject: cfg.SubjectIn, }) if err != nil { nc.Close() From 00edc35bc473df7fb9e812fe5c5f5d35774ff667 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Thu, 27 Aug 2026 16:03:02 -0300 Subject: [PATCH 54/65] Harden registrar/provider canonical matching to avoid false suppression of real NS findings. --- .../subdomain-takeover/internal/detect/registrar_helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/subdomain-takeover/internal/detect/registrar_helpers.go b/scanners/subdomain-takeover/internal/detect/registrar_helpers.go index a31f942744..435d6728e1 100644 --- a/scanners/subdomain-takeover/internal/detect/registrar_helpers.go +++ b/scanners/subdomain-takeover/internal/detect/registrar_helpers.go @@ -130,7 +130,7 @@ func canonicalProviderKey(raw string) string { } for key, aliases := range canonicalProviderAliases { for _, a := range aliases { - if s == a || strings.Contains(s, a) { + if s == a { return key } } From acd8826cc33a63cd60141fb734e4762fca34205f Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Thu, 27 Aug 2026 16:05:26 -0300 Subject: [PATCH 55/65] consistent SUBJECT_OUT and DURABLE_NAME between code and docs --- scanners/subdomain-takeover/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scanners/subdomain-takeover/README.md b/scanners/subdomain-takeover/README.md index 947c91357f..1e295a9cf3 100644 --- a/scanners/subdomain-takeover/README.md +++ b/scanners/subdomain-takeover/README.md @@ -145,8 +145,8 @@ Environment variables: - `NATS_URL` (default: `nats://localhost:4222`) - `NATS_STREAM` (default: `SCANS`) - `SUBJECT_IN` (default: `scans.dns_scanner_results`) -- `SUBJECT_OUT` (default: `scans.findings.subdomain-takeover`) -- `DURABLE_NAME` (default: `subdomain-takeover`) +- `SUBJECT_OUT` (default: `scans.findings.subdomain_takeover`) +- `DURABLE_NAME` (default: `subdomain_takeover`) - `WORKER_COUNT` (default: `5`) - `LOG_LEVEL` (default: `info`) From bce090790f8b6f281cea089d29ddc01838583bf8 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 28 Aug 2026 10:36:18 -0300 Subject: [PATCH 56/65] add https-first support for body matcher --- .../internal/detect/body_matcher.go | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/scanners/subdomain-takeover/internal/detect/body_matcher.go b/scanners/subdomain-takeover/internal/detect/body_matcher.go index b28d4b496e..aaf01d22fd 100644 --- a/scanners/subdomain-takeover/internal/detect/body_matcher.go +++ b/scanners/subdomain-takeover/internal/detect/body_matcher.go @@ -5,6 +5,7 @@ import ( "io" "net/http" "regexp" + "strconv" "strings" "time" @@ -40,13 +41,19 @@ func (m *HTTPBodyFingerprintMatcher) Contains(domain string, fingerprint string, return false } - url := fmt.Sprintf("http://%s", domain) - res, err := m.client.Get(url) + res, err := m.client.Get(fmt.Sprintf("https://%s", domain)) + if err != nil { + res, err = m.client.Get(fmt.Sprintf("http://%s", domain)) + } if err != nil { return false } defer res.Body.Close() + if expectedStatus, ok := parseHTTPStatusFingerprint(fingerprint); ok { + return res.StatusCode == expectedStatus + } + body, err := io.ReadAll(res.Body) if err != nil { return false @@ -65,3 +72,17 @@ func (m *HTTPBodyFingerprintMatcher) Contains(domain string, fingerprint string, return strings.Contains(bodyText, fingerprint) } + +func parseHTTPStatusFingerprint(fingerprint string) (int, bool) { + matches := regexp.MustCompile(`^HTTP_STATUS=(\d{3})$`).FindStringSubmatch(fingerprint) + if len(matches) != 2 { + return 0, false + } + + statusCode, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, false + } + + return statusCode, true +} From debe0f8addc17fa567dddc8a54b65186013da8d5 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 28 Aug 2026 10:39:21 -0300 Subject: [PATCH 57/65] optimize http status regex matching --- scanners/subdomain-takeover/internal/detect/body_matcher.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scanners/subdomain-takeover/internal/detect/body_matcher.go b/scanners/subdomain-takeover/internal/detect/body_matcher.go index aaf01d22fd..e710e330f6 100644 --- a/scanners/subdomain-takeover/internal/detect/body_matcher.go +++ b/scanners/subdomain-takeover/internal/detect/body_matcher.go @@ -20,6 +20,8 @@ type HTTPBodyFingerprintMatcher struct { client *http.Client } +var httpStatusFingerprintRegex = regexp.MustCompile(`^HTTP_STATUS=(\d{3})$`) + func NewHTTPBodyFingerprintMatcher(timeout time.Duration) *HTTPBodyFingerprintMatcher { return &HTTPBodyFingerprintMatcher{ client: &http.Client{Timeout: timeout}, @@ -74,7 +76,7 @@ func (m *HTTPBodyFingerprintMatcher) Contains(domain string, fingerprint string, } func parseHTTPStatusFingerprint(fingerprint string) (int, bool) { - matches := regexp.MustCompile(`^HTTP_STATUS=(\d{3})$`).FindStringSubmatch(fingerprint) + matches := httpStatusFingerprintRegex.FindStringSubmatch(fingerprint) if len(matches) != 2 { return 0, false } From 8df8ca7f036bc2c84dff7d2cc55a80f497ba015c Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 28 Aug 2026 14:23:15 -0300 Subject: [PATCH 58/65] handle benign iter.Next() errors --- scanners/subdomain-takeover/internal/app/runner.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index 924427b855..a2c3f75c2e 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -2,6 +2,7 @@ package app import ( "context" + "errors" "sync" "time" @@ -25,6 +26,13 @@ type MessageHandler interface { var checkConnection = messaging.CheckConnection +func isBenignNextError(err error) bool { + return errors.Is(err, jetstream.ErrMsgIteratorClosed) || + errors.Is(err, jetstream.ErrNoMessages) || + errors.Is(err, nats.ErrTimeout) || + errors.Is(err, context.DeadlineExceeded) +} + func Run(ctx context.Context, deps RunnerDeps) { logger := deps.Logger iter := deps.Iter @@ -58,6 +66,11 @@ Loop: logger.Info().Msg("runner stopping: context canceled") break Loop } + if isBenignNextError(err) { + nextErrCount = 0 + logger.Debug().Err(err).Msg("next returned benign idle/closed state, continuing") + continue + } nextErrCount++ if nextErrCount%10 == 0 { logger.Warn().Err(err).Int("consecutive_next_errors", nextErrCount).Msg("iterator next repeatedly failed") From ee42f6698b56acb2044d0e7fae69aa293edda5bd Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 28 Aug 2026 16:39:45 -0300 Subject: [PATCH 59/65] refactor detect/ tests and remove unneeded interfaces --- .../internal/detect/classifier.go | 25 +---- .../internal/detect/classifier_test.go | 40 +++---- .../internal/detect/cname_rules_test.go | 22 ++-- .../internal/detect/confidence_test.go | 29 +++-- .../internal/detect/evidence_test.go | 105 ++++++++++-------- .../internal/detect/fingerprint_source.go | 18 --- .../detect/fingerprint_source_test.go | 22 ---- .../internal/detect/ns_reasoning_test.go | 100 ++++++++++++----- .../internal/detect/ns_rules_test.go | 22 ++-- .../internal/detect/test_helpers_test.go | 8 -- 10 files changed, 206 insertions(+), 185 deletions(-) delete mode 100644 scanners/subdomain-takeover/internal/detect/fingerprint_source.go delete mode 100644 scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index 34ccd35246..45289b388e 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -1,26 +1,18 @@ package detect import ( + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" "github.com/rs/zerolog" ) type Classifier struct { Matcher BodyFingerprintMatcher - Source FingerprintSource logger zerolog.Logger } func NewClassifier(matcher BodyFingerprintMatcher) *Classifier { - return &Classifier{Matcher: matcher, Source: GlobalFingerprintSource{}, logger: zerolog.Nop()} -} - -func NewClassifierWithSource(matcher BodyFingerprintMatcher, source FingerprintSource) *Classifier { - if source == nil { - source = GlobalFingerprintSource{} - } - - return &Classifier{Matcher: matcher, Source: source, logger: zerolog.Nop()} + return &Classifier{Matcher: matcher, logger: zerolog.Nop()} } func (c *Classifier) WithLogger(logger zerolog.Logger) *Classifier { @@ -35,24 +27,19 @@ func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { matcher = NewNoopBodyFingerprintMatcher() } - source := c.Source - if source == nil { - source = GlobalFingerprintSource{} - } - logger := c.logger if logger.GetLevel() == zerolog.NoLevel { logger = zerolog.Nop() } - return Classify(input, matcher, source, logger) + return Classify(input, matcher, logger) } -func Classify(input model.Input, matcher BodyFingerprintMatcher, source FingerprintSource, logger zerolog.Logger) ([]model.Finding, error) { +func Classify(input model.Input, matcher BodyFingerprintMatcher, logger zerolog.Logger) ([]model.Finding, error) { findings := []model.Finding{} - cnameProviderFingerprints := source.CNAME() - nsProviderFingerprints := source.NS() + cnameProviderFingerprints := fingerprints.CNAME() + nsProviderFingerprints := fingerprints.NS() cnameEvidence := ExtractCNAMEEvidence(input.Results) if cnameEvidence != nil { diff --git a/scanners/subdomain-takeover/internal/detect/classifier_test.go b/scanners/subdomain-takeover/internal/detect/classifier_test.go index e09a055c7d..49191242f5 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier_test.go +++ b/scanners/subdomain-takeover/internal/detect/classifier_test.go @@ -9,16 +9,9 @@ import ( ) func TestClassify_ExpectedBehavior(t *testing.T) { - cnameFPs := []fingerprints.CNAMEProviderFingerprint{ - {Name: "Azure", Cname: []string{"azurewebsites.net"}, Nxdomain: true, Fingerprint: "unused"}, - {Name: "Ghost", Cname: []string{"ghost.io"}, Nxdomain: false, Fingerprint: "ghost 404", Mode: fingerprints.FingerprintModeLiteral}, + if err := fingerprints.Load(zerolog.Nop()); err != nil { + t.Fatalf("failed to load fingerprints: %v", err) } - nsFPs := []fingerprints.NSProviderFingerprint{ - {Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, - {Name: "SafeDNS", Status: fingerprints.NSStatusNotVulnerable, HostPatterns: []string{"*.safe-dns.net"}}, - } - - source := fakeSource{cname: cnameFPs, ns: nsFPs} t.Run("emits cname and ns findings when both are exploitable", func(t *testing.T) { rdapMatch := true @@ -33,7 +26,7 @@ func TestClassify_ExpectedBehavior(t *testing.T) { DelegationMatchesRDAP: &rdapMatch, }, NsDelegations: &model.NsDelegations{ - Hosts: []string{"ns1.risky-dns.net"}, + Hosts: []string{"ns1.digitalocean.com"}, Delegation: model.Delegation{ LameType: "partial", }, @@ -41,7 +34,7 @@ func TestClassify_ExpectedBehavior(t *testing.T) { }, } - findings, err := Classify(input, fakeMatcher{}, source, zerolog.Nop()) + findings, err := Classify(input, fakeMatcher{}, zerolog.Nop()) if err != nil { t.Fatalf("Classify error: %v", err) } @@ -56,12 +49,18 @@ func TestClassify_ExpectedBehavior(t *testing.T) { if f.ReasonCode != string(ReasonCNAMEDanglingNXDOMAIN) { t.Fatalf("unexpected cname reason: %q", f.ReasonCode) } + if f.Confidence != ConfidenceProbable { + t.Fatalf("unexpected cname confidence: %q", f.Confidence) + } } if f.RecordType == model.RecordTypeNS { sawNS = true if f.ReasonCode != string(ReasonNSPartialLameProviderVulnerable) { t.Fatalf("unexpected ns reason: %q", f.ReasonCode) } + if f.Confidence != ConfidenceProbable { + t.Fatalf("unexpected ns confidence: %q", f.Confidence) + } } } if !sawCNAME || !sawNS { @@ -76,7 +75,7 @@ func TestClassify_ExpectedBehavior(t *testing.T) { Domain: strPtr("b.example.ca"), CnameRecord: strPtr("b.example.ca. 300 IN CNAME foo.ghost.io."), NsDelegations: &model.NsDelegations{ - Hosts: []string{"ns1.safe-dns.net"}, + Hosts: []string{"aria.ns.cloudflare.com"}, Delegation: model.Delegation{ LameType: "full", }, @@ -88,7 +87,7 @@ func TestClassify_ExpectedBehavior(t *testing.T) { return true }} - findings, err := Classify(input, matcher, source, zerolog.Nop()) + findings, err := Classify(input, matcher, zerolog.Nop()) if err != nil { t.Fatalf("Classify error: %v", err) } @@ -99,11 +98,17 @@ func TestClassify_ExpectedBehavior(t *testing.T) { if findings[0].RecordType != model.RecordTypeCNAME { t.Fatalf("expected cname-only finding, got %s", findings[0].RecordType) } + if findings[0].ReasonCode != string(ReasonCNAMEProviderFingerprintBodyMatch) { + t.Fatalf("unexpected cname reason: %q", findings[0].ReasonCode) + } + if findings[0].Confidence != ConfidenceProbable { + t.Fatalf("unexpected cname confidence: %q", findings[0].Confidence) + } }) t.Run("returns no findings and no panic when evidence absent", func(t *testing.T) { input := model.Input{DomainKey: "k3", Results: model.ScanResults{}} - findings, err := Classify(input, fakeMatcher{}, source, zerolog.Nop()) + findings, err := Classify(input, fakeMatcher{}, zerolog.Nop()) if err != nil { t.Fatalf("Classify error: %v", err) } @@ -128,12 +133,7 @@ func TestClassifier_MethodDefaults(t *testing.T) { t.Fatalf("expected no findings, got %d", len(findings)) } - custom := NewClassifierWithSource(fakeMatcher{}, fakeSource{}) - if custom == nil { - t.Fatal("expected classifier with source") - } - - if custom.WithLogger(zerolog.Nop()) != custom { + if classifier.WithLogger(zerolog.Nop()) != classifier { t.Fatal("WithLogger should return same classifier pointer") } } diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules_test.go b/scanners/subdomain-takeover/internal/detect/cname_rules_test.go index a8c2e217c8..bb45a7338b 100644 --- a/scanners/subdomain-takeover/internal/detect/cname_rules_test.go +++ b/scanners/subdomain-takeover/internal/detect/cname_rules_test.go @@ -94,13 +94,21 @@ func TestMatchCNAMEFingerprints(t *testing.T) { } func TestShouldEmitCNAME(t *testing.T) { - if ShouldEmitCNAME(nil) { - t.Fatal("expected false for nil hit") + tests := []struct { + name string + hit *CNAMEHit + want bool + }{ + {name: "nil hit", hit: nil, want: false}, + {name: "unmatched hit", hit: &CNAMEHit{Matched: false}, want: false}, + {name: "matched hit", hit: &CNAMEHit{Matched: true}, want: true}, } - if ShouldEmitCNAME(&CNAMEHit{Matched: false}) { - t.Fatal("expected false for unmatched hit") - } - if !ShouldEmitCNAME(&CNAMEHit{Matched: true}) { - t.Fatal("expected true for matched hit") + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ShouldEmitCNAME(tt.hit); got != tt.want { + t.Fatalf("ShouldEmitCNAME()=%v want=%v", got, tt.want) + } + }) } } diff --git a/scanners/subdomain-takeover/internal/detect/confidence_test.go b/scanners/subdomain-takeover/internal/detect/confidence_test.go index d7efdd2061..ab89188fb4 100644 --- a/scanners/subdomain-takeover/internal/detect/confidence_test.go +++ b/scanners/subdomain-takeover/internal/detect/confidence_test.go @@ -4,23 +4,28 @@ import "testing" func TestConfidenceForReason(t *testing.T) { tests := []struct { + name string reason ReasonCode want string }{ - {ReasonCNAMEDanglingNXDOMAIN, ConfidenceProbable}, - {ReasonCNAMEProviderFingerprintBodyMatch, ConfidenceProbable}, - {ReasonCNAMETargetMatchMissingNXDOMAIN, ConfidenceSuspected}, - {ReasonCNAMETargetMatchMissingBodyFP, ConfidenceSuspected}, - {ReasonNSFullLameProviderVulnerable, ConfidenceConfirmed}, - {ReasonNSPartialLameProviderVulnerable, ConfidenceProbable}, - {ReasonNSLameProviderUnknown, ConfidenceSuspected}, - {ReasonNSProviderMatchOnly, ConfidenceSuspected}, - {ReasonCode("UNKNOWN_REASON"), ConfidenceSuspected}, + {name: "cname dangling nxdomain", reason: ReasonCNAMEDanglingNXDOMAIN, want: ConfidenceProbable}, + {name: "cname body fingerprint match", reason: ReasonCNAMEProviderFingerprintBodyMatch, want: ConfidenceProbable}, + {name: "cname target match missing nxdomain", reason: ReasonCNAMETargetMatchMissingNXDOMAIN, want: ConfidenceSuspected}, + {name: "cname target match missing body fp", reason: ReasonCNAMETargetMatchMissingBodyFP, want: ConfidenceSuspected}, + {name: "ns full lame vulnerable", reason: ReasonNSFullLameProviderVulnerable, want: ConfidenceConfirmed}, + {name: "ns partial lame vulnerable", reason: ReasonNSPartialLameProviderVulnerable, want: ConfidenceProbable}, + {name: "ns lame provider unknown", reason: ReasonNSLameProviderUnknown, want: ConfidenceSuspected}, + {name: "ns provider match only", reason: ReasonNSProviderMatchOnly, want: ConfidenceSuspected}, + {name: "ns registrar provider match", reason: ReasonNSRegistrarProviderMatch, want: ConfidenceSuspected}, + {name: "ns registrar context insufficient", reason: ReasonNSRegistrarContextInsufficient, want: ConfidenceSuspected}, + {name: "unknown reason defaults suspected", reason: ReasonCode("UNKNOWN_REASON"), want: ConfidenceSuspected}, } for _, tt := range tests { - if got := ConfidenceForReason(tt.reason); got != tt.want { - t.Fatalf("ConfidenceForReason(%q)=%q want=%q", tt.reason, got, tt.want) - } + t.Run(tt.name, func(t *testing.T) { + if got := ConfidenceForReason(tt.reason); got != tt.want { + t.Fatalf("ConfidenceForReason(%q)=%q want=%q", tt.reason, got, tt.want) + } + }) } } diff --git a/scanners/subdomain-takeover/internal/detect/evidence_test.go b/scanners/subdomain-takeover/internal/detect/evidence_test.go index 1dc1006fe9..c7de4d04c2 100644 --- a/scanners/subdomain-takeover/internal/detect/evidence_test.go +++ b/scanners/subdomain-takeover/internal/detect/evidence_test.go @@ -8,20 +8,27 @@ import ( ) func TestExtractCNAMEEvidence(t *testing.T) { - t.Run("returns nil when cname record missing", func(t *testing.T) { - results := minimalResults("app.example.ca") - if got := ExtractCNAMEEvidence(results); got != nil { - t.Fatalf("expected nil, got %+v", got) - } - }) + nilCases := []struct { + name string + results model.ScanResults + }{ + {name: "returns nil when cname record missing", results: minimalResults("app.example.ca")}, + { + name: "returns nil when domain missing", + results: func() model.ScanResults { + record := "a.example.ca. 300 IN CNAME foo.azurewebsites.net." + return model.ScanResults{CnameRecord: &record} + }(), + }, + } - t.Run("returns nil when domain missing", func(t *testing.T) { - record := "a.example.ca. 300 IN CNAME foo.azurewebsites.net." - results := model.ScanResults{CnameRecord: &record} - if got := ExtractCNAMEEvidence(results); got != nil { - t.Fatalf("expected nil, got %+v", got) - } - }) + for _, tt := range nilCases { + t.Run(tt.name, func(t *testing.T) { + if got := ExtractCNAMEEvidence(tt.results); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + } t.Run("extracts normalized target and noresolve true", func(t *testing.T) { record := "A.Example.CA. 300 IN CNAME Foo.AzureWebsites.NET." @@ -61,27 +68,29 @@ func TestExtractCNAMEEvidence(t *testing.T) { } func TestExtractNSEvidence(t *testing.T) { - t.Run("returns nil when domain missing", func(t *testing.T) { - input := model.ScanResults{NsDelegations: &model.NsDelegations{Hosts: []string{"ns1.example.net"}}} - if got := ExtractNSEvidence(input); got != nil { - t.Fatalf("expected nil, got %+v", got) - } - }) - - t.Run("returns nil when ns delegations missing", func(t *testing.T) { - input := minimalResults("app.example.ca") - if got := ExtractNSEvidence(input); got != nil { - t.Fatalf("expected nil, got %+v", got) - } - }) + nilCases := []struct { + name string + input model.ScanResults + }{ + {name: "returns nil when domain missing", input: model.ScanResults{NsDelegations: &model.NsDelegations{Hosts: []string{"ns1.example.net"}}}}, + {name: "returns nil when ns delegations missing", input: minimalResults("app.example.ca")}, + { + name: "returns nil when no hosts", + input: func() model.ScanResults { + input := minimalResults("app.example.ca") + input.NsDelegations = &model.NsDelegations{} + return input + }(), + }, + } - t.Run("returns nil when no hosts", func(t *testing.T) { - input := minimalResults("app.example.ca") - input.NsDelegations = &model.NsDelegations{} - if got := ExtractNSEvidence(input); got != nil { - t.Fatalf("expected nil, got %+v", got) - } - }) + for _, tt := range nilCases { + t.Run(tt.name, func(t *testing.T) { + if got := ExtractNSEvidence(tt.input); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + } t.Run("normalizes, sorts, and deduplicates hosts", func(t *testing.T) { input := minimalResults("app.example.ca") @@ -111,18 +120,24 @@ func TestExtractNSEvidence(t *testing.T) { } func TestNormalizeHostAndParseHelpers(t *testing.T) { - if got := normalizeHost(".NS1.Example.NET."); got != "ns1.example.net" { - t.Fatalf("normalizeHost mismatch: %q", got) - } + t.Run("normalizeHost trims dots and lowercases", func(t *testing.T) { + if got := normalizeHost(".NS1.Example.NET."); got != "ns1.example.net" { + t.Fatalf("normalizeHost mismatch: %q", got) + } + }) - record := "x.example.ca. 300 IN CNAME target.provider.net." - if got := parseCname(record); got != "target.provider.net" { - t.Fatalf("parseCname mismatch: %q", got) - } + t.Run("parseCname extracts normalized target", func(t *testing.T) { + record := "x.example.ca. 300 IN CNAME target.provider.net." + if got := parseCname(record); got != "target.provider.net" { + t.Fatalf("parseCname mismatch: %q", got) + } + }) - hosts := []string{"ns3.example.net", "NS1.example.net.", "ns1.example.net", "ns2.example.net"} - expected := []string{"ns1.example.net", "ns2.example.net", "ns3.example.net"} - if got := parseHostnames(hosts); !reflect.DeepEqual(got, expected) { - t.Fatalf("parseHostnames mismatch: got=%v want=%v", got, expected) - } + t.Run("parseHostnames sorts and deduplicates", func(t *testing.T) { + hosts := []string{"ns3.example.net", "NS1.example.net.", "ns1.example.net", "ns2.example.net"} + expected := []string{"ns1.example.net", "ns2.example.net", "ns3.example.net"} + if got := parseHostnames(hosts); !reflect.DeepEqual(got, expected) { + t.Fatalf("parseHostnames mismatch: got=%v want=%v", got, expected) + } + }) } diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_source.go b/scanners/subdomain-takeover/internal/detect/fingerprint_source.go deleted file mode 100644 index b6b202e1d7..0000000000 --- a/scanners/subdomain-takeover/internal/detect/fingerprint_source.go +++ /dev/null @@ -1,18 +0,0 @@ -package detect - -import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" - -type FingerprintSource interface { - CNAME() []fingerprints.CNAMEProviderFingerprint - NS() []fingerprints.NSProviderFingerprint -} - -type GlobalFingerprintSource struct{} - -func (GlobalFingerprintSource) CNAME() []fingerprints.CNAMEProviderFingerprint { - return fingerprints.CNAME() -} - -func (GlobalFingerprintSource) NS() []fingerprints.NSProviderFingerprint { - return fingerprints.NS() -} diff --git a/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go b/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go deleted file mode 100644 index 34f45f1c3b..0000000000 --- a/scanners/subdomain-takeover/internal/detect/fingerprint_source_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package detect - -import ( - "testing" - - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" - "github.com/rs/zerolog" -) - -func TestGlobalFingerprintSource(t *testing.T) { - if err := fingerprints.Load(zerolog.Nop()); err != nil { - t.Fatalf("failed to load fingerprints: %v", err) - } - - src := GlobalFingerprintSource{} - if got := src.CNAME(); len(got) == 0 { - t.Fatal("expected cname source to be populated") - } - if got := src.NS(); len(got) == 0 { - t.Fatal("expected ns source to be populated") - } -} diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go index 3b8fccb923..742e510002 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go @@ -88,34 +88,80 @@ func TestGetNSHijackReasonCode(t *testing.T) { } func TestNSReasoningHelpers(t *testing.T) { - if !isExploitableProviderStatus(fingerprints.NSStatusVulnerable) { - t.Fatal("expected vulnerable to be exploitable") - } - if !isExploitableProviderStatus(fingerprints.NSStatusVulnerableWithPurchase) { - t.Fatal("expected vulnerable_with_purchase to be exploitable") - } - if isExploitableProviderStatus(fingerprints.NSStatusNotVulnerable) { - t.Fatal("expected not_vulnerable to be non-exploitable") - } + t.Run("isExploitableProviderStatus", func(t *testing.T) { + tests := []struct { + name string + status fingerprints.NSProviderStatus + want bool + }{ + {name: "vulnerable", status: fingerprints.NSStatusVulnerable, want: true}, + {name: "vulnerable with purchase", status: fingerprints.NSStatusVulnerableWithPurchase, want: true}, + {name: "not vulnerable", status: fingerprints.NSStatusNotVulnerable, want: false}, + } - if got := normalizeLameType(" Partial "); got != "partial" { - t.Fatalf("normalizeLameType mismatch: %q", got) - } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isExploitableProviderStatus(tt.status); got != tt.want { + t.Fatalf("isExploitableProviderStatus(%q)=%v want=%v", tt.status, got, tt.want) + } + }) + } + }) - if !isRegistrarMismatch("Digital Ocean", &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"}) { - t.Fatal("expected registrar mismatch for different providers") - } - if isRegistrarMismatch("Digital Ocean", &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."}) { - t.Fatal("expected same provider to fail mismatch check") - } + t.Run("normalizeLameType", func(t *testing.T) { + if got := normalizeLameType(" Partial "); got != "partial" { + t.Fatalf("normalizeLameType mismatch: %q", got) + } + }) - if nsReasonRank(ReasonNSFullLameProviderVulnerable) <= nsReasonRank(ReasonNSPartialLameProviderVulnerable) { - t.Fatal("expected full lame rank > partial lame rank") - } - if nsReasonRank(ReasonNSPartialLameProviderVulnerable) <= nsReasonRank(ReasonNSRegistrarProviderMatch) { - t.Fatal("expected partial lame rank > registrar-provider-match rank") - } - if nsReasonRank(ReasonNSRegistrarProviderMatch) <= nsReasonRank(ReasonNSProviderMatchOnly) { - t.Fatal("expected registrar gate rank > provider-only rank") - } + t.Run("isRegistrarMismatch", func(t *testing.T) { + tests := []struct { + name string + provider string + registrar *model.RegistrarContext + want bool + }{ + { + name: "different providers", + provider: "Digital Ocean", + registrar: &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"}, + want: true, + }, + { + name: "same provider", + provider: "Digital Ocean", + registrar: &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isRegistrarMismatch(tt.provider, tt.registrar); got != tt.want { + t.Fatalf("isRegistrarMismatch()=%v want=%v", got, tt.want) + } + }) + } + }) + + t.Run("nsReasonRank ordering", func(t *testing.T) { + checks := []struct { + name string + high ReasonCode + low ReasonCode + label string + }{ + {name: "full above partial", high: ReasonNSFullLameProviderVulnerable, low: ReasonNSPartialLameProviderVulnerable, label: "full lame > partial lame"}, + {name: "partial above registrar-provider-match", high: ReasonNSPartialLameProviderVulnerable, low: ReasonNSRegistrarProviderMatch, label: "partial lame > registrar-provider-match"}, + {name: "registrar gate above provider only", high: ReasonNSRegistrarProviderMatch, low: ReasonNSProviderMatchOnly, label: "registrar gate > provider-only"}, + } + + for _, check := range checks { + t.Run(check.name, func(t *testing.T) { + if nsReasonRank(check.high) <= nsReasonRank(check.low) { + t.Fatalf("expected %s", check.label) + } + }) + } + }) } diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go index 61b41f7e47..15167d8622 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go +++ b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go @@ -148,13 +148,21 @@ func TestMatchNSProviderRules(t *testing.T) { } func TestShouldEmitNSHijack(t *testing.T) { - if ShouldEmitNSHijack(nil) { - t.Fatal("expected false for nil hit") + tests := []struct { + name string + hit *NSHit + want bool + }{ + {name: "nil hit", hit: nil, want: false}, + {name: "unmatched hit", hit: &NSHit{Matched: false}, want: false}, + {name: "matched hit", hit: &NSHit{Matched: true}, want: true}, } - if ShouldEmitNSHijack(&NSHit{Matched: false}) { - t.Fatal("expected false for unmatched hit") - } - if !ShouldEmitNSHijack(&NSHit{Matched: true}) { - t.Fatal("expected true for matched hit") + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ShouldEmitNSHijack(tt.hit); got != tt.want { + t.Fatalf("ShouldEmitNSHijack()=%v want=%v", got, tt.want) + } + }) } } diff --git a/scanners/subdomain-takeover/internal/detect/test_helpers_test.go b/scanners/subdomain-takeover/internal/detect/test_helpers_test.go index 057f7101eb..b20cfb1155 100644 --- a/scanners/subdomain-takeover/internal/detect/test_helpers_test.go +++ b/scanners/subdomain-takeover/internal/detect/test_helpers_test.go @@ -16,14 +16,6 @@ func (f fakeMatcher) Contains(domain string, fingerprint string, mode fingerprin return f.containsFn(domain, fingerprint, mode) } -type fakeSource struct { - cname []fingerprints.CNAMEProviderFingerprint - ns []fingerprints.NSProviderFingerprint -} - -func (f fakeSource) CNAME() []fingerprints.CNAMEProviderFingerprint { return f.cname } -func (f fakeSource) NS() []fingerprints.NSProviderFingerprint { return f.ns } - func strPtr(v string) *string { return &v } func boolPtr(v bool) *bool { return &v } From 86c78d4775fd9435f65cdf49632c2c5623897669 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 28 Aug 2026 16:40:18 -0300 Subject: [PATCH 60/65] refactor app/ tests and remove unneeded interfaces --- .../subdomain-takeover/cmd/service/main.go | 4 +- .../subdomain-takeover/internal/app/runner.go | 6 +- .../internal/app/runner_test.go | 39 ++-- .../subdomain-takeover/internal/app/worker.go | 30 +-- .../internal/app/worker_test.go | 219 +++++++++--------- .../internal/messaging/publisher.go | 20 +- .../internal/messaging/publisher_test.go | 206 ++++++++-------- 7 files changed, 256 insertions(+), 268 deletions(-) diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 816ab0cc80..93047d8f67 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -53,10 +53,10 @@ func main() { logger.Info().Msgf("Connected to NATS at %s", runtimeDeps.NC.ConnectedUrl()) - pub := messaging.NewPublisher(logger, runtimeDeps.JS, cfg.SubjectOut) + pub := messaging.NewPublisher(logger, runtimeDeps.JS.Publish, cfg.SubjectOut) matcher := detect.NewHTTPBodyFingerprintMatcher(5 * time.Second) classifier := detect.NewClassifier(matcher).WithLogger(logger) - worker := app.NewWorker(logger, pub, classifier) + worker := app.NewWorker(logger, pub.Publish, classifier.Classify) go func() { <-sig diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index a2c3f75c2e..2d3547373b 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -16,14 +16,10 @@ type RunnerDeps struct { Logger zerolog.Logger WorkerCount int Iter jetstream.MessagesContext - Worker MessageHandler + Worker *Worker NC *nats.Conn } -type MessageHandler interface { - Handle(ctx context.Context, msg jetstream.Msg) error -} - var checkConnection = messaging.CheckConnection func isBenignNextError(err error) bool { diff --git a/scanners/subdomain-takeover/internal/app/runner_test.go b/scanners/subdomain-takeover/internal/app/runner_test.go index b732ecd76a..be061d6bae 100644 --- a/scanners/subdomain-takeover/internal/app/runner_test.go +++ b/scanners/subdomain-takeover/internal/app/runner_test.go @@ -38,35 +38,23 @@ func (f *fakeMessagesIter) Next(...jetstream.NextOpt) (jetstream.Msg, error) { func (f *fakeMessagesIter) Stop() { f.stopCalled = true } func (f *fakeMessagesIter) Drain() {} -type fakeHandler struct { - mu sync.Mutex - count int -} - -func (f *fakeHandler) Handle(context.Context, jetstream.Msg) error { - f.mu.Lock() - f.count++ - f.mu.Unlock() - return nil -} - // nolint:revive // nats Conn fields are unexported; this fake nil connection path only. func TestRun_ReturnsImmediatelyWhenConnectionUnhealthy(t *testing.T) { iter := &fakeMessagesIter{} - h := &fakeHandler{} + worker := NewWorker(zerolog.Nop(), (&fakePublisher{}).Publish, (fakeClassifier{}).Classify) deps := RunnerDeps{ Logger: zerolog.Nop(), WorkerCount: 2, Iter: iter, - Worker: h, + Worker: worker, NC: nil, } Run(context.Background(), deps) - if h.count != 0 { - t.Fatalf("expected no handled messages, got %d", h.count) + if iter.idx != 0 { + t.Fatalf("expected no iterator reads, got %d", iter.idx) } } @@ -83,20 +71,21 @@ func TestRun_ClampsWorkerCountBelowOne(t *testing.T) { return nil } - iter := &fakeMessagesIter{msgs: []jetstream.Msg{&fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"}}} - h := &fakeHandler{} + msg := &fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"} + iter := &fakeMessagesIter{msgs: []jetstream.Msg{msg}} + worker := NewWorker(zerolog.Nop(), (&fakePublisher{}).Publish, (fakeClassifier{}).Classify) deps := RunnerDeps{ Logger: zerolog.Nop(), WorkerCount: 0, Iter: iter, - Worker: h, + Worker: worker, NC: nil, } Run(context.Background(), deps) - if h.count != 1 { - t.Fatalf("expected one handled message, got %d", h.count) + if msg.ackCount != 1 { + t.Fatalf("expected one handled message (ack once), got ack=%d", msg.ackCount) } } @@ -106,7 +95,7 @@ func TestRun_ExitsWhenContextCancelledDuringNextErrors(t *testing.T) { checkConnection = func(_ *nats.Conn) error { return nil } iter := &fakeMessagesIter{err: errors.New("next failed")} - h := &fakeHandler{} + worker := NewWorker(zerolog.Nop(), (&fakePublisher{}).Publish, (fakeClassifier{}).Classify) ctx, cancel := context.WithCancel(context.Background()) go func() { @@ -118,13 +107,13 @@ func TestRun_ExitsWhenContextCancelledDuringNextErrors(t *testing.T) { Logger: zerolog.Nop(), WorkerCount: 1, Iter: iter, - Worker: h, + Worker: worker, NC: nil, } Run(ctx, deps) - if h.count != 0 { - t.Fatalf("expected no handled messages, got %d", h.count) + if iter.idx != 0 { + t.Fatalf("expected no iterator reads, got %d", iter.idx) } } diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index eca59f91bc..6bdc81960f 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -11,22 +11,22 @@ import ( "github.com/rs/zerolog" ) -type FindingPublisher interface { - Publish(ctx context.Context, finding model.Finding) error -} - -type ScanClassifier interface { - Classify(input model.Input) ([]model.Finding, error) -} - type Worker struct { - logger zerolog.Logger - publisher FindingPublisher - classifier ScanClassifier + logger zerolog.Logger + publishFinding func(ctx context.Context, finding model.Finding) error + classifyFinding func(input model.Input) ([]model.Finding, error) } -func NewWorker(logger zerolog.Logger, publisher FindingPublisher, classifier ScanClassifier) *Worker { - return &Worker{logger: logger, publisher: publisher, classifier: classifier} +func NewWorker( + logger zerolog.Logger, + publishFinding func(ctx context.Context, finding model.Finding) error, + classifyFinding func(input model.Input) ([]model.Finding, error), +) *Worker { + return &Worker{ + logger: logger, + publishFinding: publishFinding, + classifyFinding: classifyFinding, + } } func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { @@ -45,7 +45,7 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { log = log.With().Str("domain_key", scan.DomainKey).Logger() - findings, err := w.classifier.Classify(scan) + findings, err := w.classifyFinding(scan) if err != nil { log.Err(err).Msg("classify error") w.nak(msg, log, err) @@ -57,7 +57,7 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { } for _, finding := range findings { - err = w.publisher.Publish(ctx, finding) + err = w.publishFinding(ctx, finding) if err != nil { log.Err(err).Msg("publish error") w.nak(msg, log, err) diff --git a/scanners/subdomain-takeover/internal/app/worker_test.go b/scanners/subdomain-takeover/internal/app/worker_test.go index fc771e4e15..8a41806b2c 100644 --- a/scanners/subdomain-takeover/internal/app/worker_test.go +++ b/scanners/subdomain-takeover/internal/app/worker_test.go @@ -81,120 +81,111 @@ func (m *fakeJSMsg) TermWithReason(string) error { return nil } func TestWorkerHandle(t *testing.T) { logger := zerolog.Nop() - t.Run("decode error terminates message", func(t *testing.T) { - pub := &fakePublisher{} - classifier := fakeClassifier{} - worker := NewWorker(logger, pub, classifier) - - msg := &fakeJSMsg{data: []byte("{not-json"), subject: "scans.dns_scanner_results"} - err := worker.Handle(context.Background(), msg) - if err == nil { - t.Fatal("expected decode error") - } - if msg.termCount != 1 { - t.Fatalf("expected term once, got %d", msg.termCount) - } - if msg.nakCount != 0 || msg.ackCount != 0 { - t.Fatalf("unexpected ack/nak counts: ack=%d nak=%d", msg.ackCount, msg.nakCount) - } - }) - - t.Run("classification error naks message", func(t *testing.T) { - pub := &fakePublisher{} - classifier := fakeClassifier{err: errors.New("classify failed")} - worker := NewWorker(logger, pub, classifier) - - msg := &fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"} - err := worker.Handle(context.Background(), msg) - if err == nil { - t.Fatal("expected classification error") - } - if msg.nakCount != 1 { - t.Fatalf("expected nak once, got %d", msg.nakCount) - } - if msg.ackCount != 0 || msg.termCount != 0 { - t.Fatalf("unexpected ack/term counts: ack=%d term=%d", msg.ackCount, msg.termCount) - } - }) - - t.Run("publish error naks message", func(t *testing.T) { - pub := &fakePublisher{err: errors.New("publish failed")} - classifier := fakeClassifier{findings: []model.Finding{{Domain: "a.example.ca"}}} - worker := NewWorker(logger, pub, classifier) - - msg := &fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"} - err := worker.Handle(context.Background(), msg) - if err == nil { - t.Fatal("expected publish error") - } - if msg.nakCount != 1 { - t.Fatalf("expected nak once, got %d", msg.nakCount) - } - if msg.ackCount != 0 || msg.termCount != 0 { - t.Fatalf("unexpected ack/term counts: ack=%d term=%d", msg.ackCount, msg.termCount) - } - }) - - t.Run("successful processing publishes all findings and acks", func(t *testing.T) { - pub := &fakePublisher{} - classifier := fakeClassifier{findings: []model.Finding{{Domain: "a.example.ca"}, {Domain: "b.example.ca"}}} - worker := NewWorker(logger, pub, classifier) - - msg := &fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"} - err := worker.Handle(context.Background(), msg) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(pub.published) != 2 { - t.Fatalf("expected 2 published findings, got %d", len(pub.published)) - } - if msg.ackCount != 1 { - t.Fatalf("expected ack once, got %d", msg.ackCount) - } - if msg.nakCount != 0 || msg.termCount != 0 { - t.Fatalf("unexpected nak/term counts: nak=%d term=%d", msg.nakCount, msg.termCount) - } - }) - - t.Run("ack failure returns error", func(t *testing.T) { - pub := &fakePublisher{} - classifier := fakeClassifier{} - worker := NewWorker(logger, pub, classifier) - - msg := &fakeJSMsg{ - data: []byte(`{"domain_key":"k","results":{}}`), - subject: "scans.dns_scanner_results", - ackErr: errors.New("ack failed"), - } - - err := worker.Handle(context.Background(), msg) - if err == nil { - t.Fatal("expected ack error") - } - if msg.ackCount != 1 { - t.Fatalf("expected ack once, got %d", msg.ackCount) - } - }) - - t.Run("decode error still returned when term fails", func(t *testing.T) { - pub := &fakePublisher{} - classifier := fakeClassifier{} - worker := NewWorker(logger, pub, classifier) - - msg := &fakeJSMsg{ - data: []byte("{bad-json"), - subject: "scans.dns_scanner_results", - termErr: errors.New("term failed"), - } - - err := worker.Handle(context.Background(), msg) - if err == nil { - t.Fatal("expected decode error") - } - if msg.termCount != 1 { - t.Fatalf("expected term once, got %d", msg.termCount) - } - }) + tests := []struct { + name string + data []byte + ackErr error + nakErr error + termErr error + classifier fakeClassifier + publisherErr error + wantErr bool + wantAck int + wantNak int + wantTerm int + wantPublishedFindings int + }{ + { + name: "decode error terminates message", + data: []byte("{not-json"), + wantErr: true, + wantAck: 0, + wantNak: 0, + wantTerm: 1, + }, + { + name: "classification error naks message", + data: []byte(`{"domain_key":"k","results":{}}`), + classifier: fakeClassifier{err: errors.New("classify failed")}, + wantErr: true, + wantAck: 0, + wantNak: 1, + wantTerm: 0, + }, + { + name: "publish error naks message", + data: []byte(`{"domain_key":"k","results":{}}`), + classifier: fakeClassifier{findings: []model.Finding{{Domain: "a.example.ca"}}}, + publisherErr: errors.New("publish failed"), + wantErr: true, + wantAck: 0, + wantNak: 1, + wantTerm: 0, + }, + { + name: "successful processing publishes all findings and acks", + data: []byte(`{"domain_key":"k","results":{}}`), + classifier: fakeClassifier{findings: []model.Finding{{Domain: "a.example.ca"}, {Domain: "b.example.ca"}}}, + wantErr: false, + wantAck: 1, + wantNak: 0, + wantTerm: 0, + wantPublishedFindings: 2, + }, + { + name: "ack failure returns error", + data: []byte(`{"domain_key":"k","results":{}}`), + ackErr: errors.New("ack failed"), + wantErr: true, + wantAck: 1, + wantNak: 0, + wantTerm: 0, + }, + { + name: "decode error still returned when term fails", + data: []byte("{bad-json"), + termErr: errors.New("term failed"), + wantErr: true, + wantAck: 0, + wantNak: 0, + wantTerm: 1, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + pub := &fakePublisher{err: tc.publisherErr} + worker := NewWorker(logger, pub.Publish, tc.classifier.Classify) + + msg := &fakeJSMsg{ + data: tc.data, + subject: "scans.dns_scanner_results", + ackErr: tc.ackErr, + nakErr: tc.nakErr, + termErr: tc.termErr, + } + + err := worker.Handle(context.Background(), msg) + if tc.wantErr && err == nil { + t.Fatal("expected error") + } + if !tc.wantErr && err != nil { + t.Fatalf("unexpected error: %v", err) + } + if msg.ackCount != tc.wantAck { + t.Fatalf("unexpected ack count: got=%d want=%d", msg.ackCount, tc.wantAck) + } + if msg.nakCount != tc.wantNak { + t.Fatalf("unexpected nak count: got=%d want=%d", msg.nakCount, tc.wantNak) + } + if msg.termCount != tc.wantTerm { + t.Fatalf("unexpected term count: got=%d want=%d", msg.termCount, tc.wantTerm) + } + if len(pub.published) != tc.wantPublishedFindings { + t.Fatalf("unexpected published findings count: got=%d want=%d", len(pub.published), tc.wantPublishedFindings) + } + }) + } } func TestDecodeScan_TrimsTrailingNewline(t *testing.T) { diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go index 9083980cf9..bd43b9c2d4 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -15,17 +15,17 @@ var marshalFinding = json.Marshal var nowUTC = func() time.Time { return time.Now().UTC() } type Publisher struct { - logger zerolog.Logger - js publishClient - subject string + logger zerolog.Logger + publishToNATS func(ctx context.Context, subj string, data []byte, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error) + subject string } -type publishClient interface { - Publish(ctx context.Context, subj string, data []byte, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error) -} - -func NewPublisher(logger zerolog.Logger, js publishClient, subject string) *Publisher { - return &Publisher{logger: logger, js: js, subject: subject} +func NewPublisher( + logger zerolog.Logger, + publishToNATS func(ctx context.Context, subj string, data []byte, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error), + subject string, +) *Publisher { + return &Publisher{logger: logger, publishToNATS: publishToNATS, subject: subject} } func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { @@ -54,7 +54,7 @@ func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { return err } - if _, err := p.js.Publish(ctx, p.subject, payload); err != nil { + if _, err := p.publishToNATS(ctx, p.subject, payload); err != nil { p.logger.Error(). Err(err). Str("domain", finding.Domain). diff --git a/scanners/subdomain-takeover/internal/messaging/publisher_test.go b/scanners/subdomain-takeover/internal/messaging/publisher_test.go index ca19ea62d2..785a372663 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher_test.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher_test.go @@ -35,7 +35,7 @@ func (f *fakePublishClient) Publish(_ context.Context, subj string, data []byte, func TestNewPublisher(t *testing.T) { client := &fakePublishClient{} - p := NewPublisher(zerolog.Nop(), client, "scans.findings.subdomain-takeover") + p := NewPublisher(zerolog.Nop(), client.Publish, "scans.findings.subdomain-takeover") if p == nil { t.Fatal("expected publisher instance") } @@ -72,76 +72,79 @@ func TestPublisherPublish(t *testing.T) { ReasonCode: "CNAME_DANGLING_NXDOMAIN", } - t.Run("publishes ns finding event payload", func(t *testing.T) { - client := &fakePublishClient{} - p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain-takeover"} - - err := p.Publish(context.Background(), nsFinding) - if err != nil { - t.Fatalf("unexpected publish error: %v", err) - } - if client.called != 1 { - t.Fatalf("expected one publish call, got %d", client.called) - } - if client.lastSubj != "scans.findings.subdomain-takeover" { - t.Fatalf("unexpected subject: %q", client.lastSubj) - } - - var got model.FindingEvent - if err := json.Unmarshal(client.lastBytes, &got); err != nil { - t.Fatalf("payload not valid json: %v", err) - } - - if got.Source != "subdomain-takeover" { - t.Fatalf("unexpected source: %q", got.Source) - } - if got.FindingType != "subdomain-takeover-ns" { - t.Fatalf("unexpected findingType: %q", got.FindingType) - } - if got.DomainKey != nsFinding.DomainKey { - t.Fatalf("unexpected domainKey: %q", got.DomainKey) - } - if got.Subject != nsFinding.Domain { - t.Fatalf("unexpected subject: %q", got.Subject) - } - if got.ObservedAt != fixedNow.Format(time.RFC3339) { - t.Fatalf("unexpected observedAt: %q", got.ObservedAt) - } - if got.Evidence["target"] != nsFinding.Target { - t.Fatalf("unexpected evidence target: %v", got.Evidence["target"]) - } - if got.Evidence["recordType"] != string(nsFinding.RecordType) { - t.Fatalf("unexpected evidence recordType: %v", got.Evidence["recordType"]) - } - if got.Attributes["provider"] != nsFinding.Provider { - t.Fatalf("unexpected attributes provider: %v", got.Attributes["provider"]) - } - if got.Attributes["lameType"] != nsFinding.LameType { - t.Fatalf("unexpected attributes lameType: %v", got.Attributes["lameType"]) - } - }) - - t.Run("maps cname record to cname findingType", func(t *testing.T) { - client := &fakePublishClient{} - p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain-takeover"} - - err := p.Publish(context.Background(), cnameFinding) - if err != nil { - t.Fatalf("unexpected publish error: %v", err) - } - - var got model.FindingEvent - if err := json.Unmarshal(client.lastBytes, &got); err != nil { - t.Fatalf("payload not valid json: %v", err) - } - if got.FindingType != "subdomain-takeover-cname" { - t.Fatalf("unexpected findingType: %q", got.FindingType) + t.Run("publishes expected finding event payload", func(t *testing.T) { + tests := []struct { + name string + finding model.Finding + wantFindingType string + }{ + { + name: "ns record", + finding: nsFinding, + wantFindingType: "subdomain-takeover-ns", + }, + { + name: "cname record", + finding: cnameFinding, + wantFindingType: "subdomain-takeover-cname", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + client := &fakePublishClient{} + p := &Publisher{logger: zerolog.Nop(), publishToNATS: client.Publish, subject: "scans.findings.subdomain-takeover"} + + err := p.Publish(context.Background(), tc.finding) + if err != nil { + t.Fatalf("unexpected publish error: %v", err) + } + if client.called != 1 { + t.Fatalf("expected one publish call, got %d", client.called) + } + if client.lastSubj != "scans.findings.subdomain-takeover" { + t.Fatalf("unexpected subject: %q", client.lastSubj) + } + + var got model.FindingEvent + if err := json.Unmarshal(client.lastBytes, &got); err != nil { + t.Fatalf("payload not valid json: %v", err) + } + + if got.Source != "subdomain-takeover" { + t.Fatalf("unexpected source: %q", got.Source) + } + if got.FindingType != tc.wantFindingType { + t.Fatalf("unexpected findingType: %q", got.FindingType) + } + if got.DomainKey != tc.finding.DomainKey { + t.Fatalf("unexpected domainKey: %q", got.DomainKey) + } + if got.Subject != tc.finding.Domain { + t.Fatalf("unexpected subject: %q", got.Subject) + } + if got.ObservedAt != fixedNow.Format(time.RFC3339) { + t.Fatalf("unexpected observedAt: %q", got.ObservedAt) + } + if got.Evidence["target"] != tc.finding.Target { + t.Fatalf("unexpected evidence target: %v", got.Evidence["target"]) + } + if got.Evidence["recordType"] != string(tc.finding.RecordType) { + t.Fatalf("unexpected evidence recordType: %v", got.Evidence["recordType"]) + } + if got.Attributes["provider"] != tc.finding.Provider { + t.Fatalf("unexpected attributes provider: %v", got.Attributes["provider"]) + } + if got.Attributes["lameType"] != tc.finding.LameType { + t.Fatalf("unexpected attributes lameType: %v", got.Attributes["lameType"]) + } + }) } }) - t.Run("returns publish client error", func(t *testing.T) { + t.Run("propagates publish client error", func(t *testing.T) { client := &fakePublishClient{err: errors.New("publish failed")} - p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain-takeover"} + p := &Publisher{logger: zerolog.Nop(), publishToNATS: client.Publish, subject: "scans.findings.subdomain-takeover"} err := p.Publish(context.Background(), nsFinding) if err == nil { @@ -152,36 +155,45 @@ func TestPublisherPublish(t *testing.T) { } }) - t.Run("returns marshal error and does not publish", func(t *testing.T) { - marshalFinding = func(v any) ([]byte, error) { - return nil, errors.New("marshal failed") - } - - client := &fakePublishClient{} - p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain-takeover"} - - err := p.Publish(context.Background(), nsFinding) - if err == nil { - t.Fatal("expected marshal error") - } - if client.called != 0 { - t.Fatalf("expected no publish call, got %d", client.called) - } - }) - - t.Run("returns mapping error for unsupported record type", func(t *testing.T) { - client := &fakePublishClient{} - p := &Publisher{logger: zerolog.Nop(), js: client, subject: "scans.findings.subdomain-takeover"} - - badFinding := nsFinding - badFinding.RecordType = model.RecordType("TXT") - - err := p.Publish(context.Background(), badFinding) - if err == nil { - t.Fatal("expected mapping error") - } - if client.called != 0 { - t.Fatalf("expected no publish call, got %d", client.called) + t.Run("returns mapping or marshal errors without publish", func(t *testing.T) { + tests := []struct { + name string + finding model.Finding + marshalErr error + }{ + { + name: "marshal error", + finding: nsFinding, + marshalErr: errors.New("marshal failed"), + }, + { + name: "unsupported record type mapping error", + finding: func() model.Finding { + badFinding := nsFinding + badFinding.RecordType = model.RecordType("TXT") + return badFinding + }(), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + marshalFinding = origMarshal + if tc.marshalErr != nil { + marshalFinding = func(any) ([]byte, error) { return nil, tc.marshalErr } + } + + client := &fakePublishClient{} + p := &Publisher{logger: zerolog.Nop(), publishToNATS: client.Publish, subject: "scans.findings.subdomain-takeover"} + + err := p.Publish(context.Background(), tc.finding) + if err == nil { + t.Fatal("expected error") + } + if client.called != 0 { + t.Fatalf("expected no publish call, got %d", client.called) + } + }) } }) } From 872bf82259748e4dac1729ad630f8e3bf24ba62e Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Fri, 28 Aug 2026 16:51:38 -0300 Subject: [PATCH 61/65] remove wrappers for nak and term functions in worker --- .../subdomain-takeover/internal/app/worker.go | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index 6bdc81960f..79b59fa57e 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -39,7 +39,9 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { scan, err := decodeScan(msg.Data()) if err != nil { log.Err(err).Msg("decode error") - w.term(msg, log, err) + if termErr := msg.Term(); termErr != nil { + log.Error().Err(fmt.Errorf("original=%v term=%w", err, termErr)).Msg("failed to term message") + } return err } @@ -48,7 +50,9 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { findings, err := w.classifyFinding(scan) if err != nil { log.Err(err).Msg("classify error") - w.nak(msg, log, err) + if nakErr := msg.Nak(); nakErr != nil { + log.Error().Err(fmt.Errorf("original=%v nak=%w", err, nakErr)).Msg("failed to nak message") + } return err } @@ -60,7 +64,9 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { err = w.publishFinding(ctx, finding) if err != nil { log.Err(err).Msg("publish error") - w.nak(msg, log, err) + if nakErr := msg.Nak(); nakErr != nil { + log.Error().Err(fmt.Errorf("original=%v nak=%w", err, nakErr)).Msg("failed to nak message") + } return err } } @@ -81,15 +87,3 @@ func decodeScan(data []byte) (model.Input, error) { err := json.Unmarshal(trimmed, &scan) return scan, err } - -func (w *Worker) nak(msg jetstream.Msg, log zerolog.Logger, originalErr error) { - if err := msg.Nak(); err != nil { - log.Error().Err(fmt.Errorf("original=%v nak=%w", originalErr, err)).Msg("failed to nak message") - } -} - -func (w *Worker) term(msg jetstream.Msg, log zerolog.Logger, originalErr error) { - if err := msg.Term(); err != nil { - log.Error().Err(fmt.Errorf("original=%v term=%w", originalErr, err)).Msg("failed to term message") - } -} From 04c14a70a22714b94c6bb4abfc1c013b8be0bf19 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Mon, 31 Aug 2026 11:04:21 -0300 Subject: [PATCH 62/65] refactor(subdomain-takeover): remove test-only seams from prod code --- .../subdomain-takeover/cmd/service/main.go | 6 +- .../subdomain-takeover/internal/app/runner.go | 4 +- .../internal/app/runner_test.go | 119 ----------- .../subdomain-takeover/internal/app/worker.go | 24 +-- .../internal/app/worker_test.go | 196 ----------------- .../internal/detect/classifier.go | 49 ++--- .../internal/detect/classifier_test.go | 139 ------------ .../internal/detect/cname_rules.go | 17 +- .../internal/detect/cname_rules_test.go | 114 ---------- .../internal/detect/logger.go | 9 - .../internal/detect/logger_test.go | 15 -- .../internal/detect/ns_reasoning.go | 4 - .../internal/detect/ns_reasoning_test.go | 167 --------------- .../internal/detect/ns_rules.go | 15 +- .../internal/detect/ns_rules_test.go | 168 --------------- .../internal/detect/registrar_helpers.go | 132 ++++++------ .../internal/messaging/health.go | 11 - .../internal/messaging/health_test.go | 52 ----- .../internal/messaging/publisher.go | 23 +- .../internal/messaging/publisher_test.go | 199 ------------------ .../internal/model/finding.go | 13 +- 21 files changed, 130 insertions(+), 1346 deletions(-) delete mode 100644 scanners/subdomain-takeover/internal/app/runner_test.go delete mode 100644 scanners/subdomain-takeover/internal/app/worker_test.go delete mode 100644 scanners/subdomain-takeover/internal/detect/classifier_test.go delete mode 100644 scanners/subdomain-takeover/internal/detect/cname_rules_test.go delete mode 100644 scanners/subdomain-takeover/internal/detect/logger.go delete mode 100644 scanners/subdomain-takeover/internal/detect/logger_test.go delete mode 100644 scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go delete mode 100644 scanners/subdomain-takeover/internal/detect/ns_rules_test.go delete mode 100644 scanners/subdomain-takeover/internal/messaging/health_test.go delete mode 100644 scanners/subdomain-takeover/internal/messaging/publisher_test.go diff --git a/scanners/subdomain-takeover/cmd/service/main.go b/scanners/subdomain-takeover/cmd/service/main.go index 93047d8f67..c72b229142 100644 --- a/scanners/subdomain-takeover/cmd/service/main.go +++ b/scanners/subdomain-takeover/cmd/service/main.go @@ -53,10 +53,10 @@ func main() { logger.Info().Msgf("Connected to NATS at %s", runtimeDeps.NC.ConnectedUrl()) - pub := messaging.NewPublisher(logger, runtimeDeps.JS.Publish, cfg.SubjectOut) + pub := messaging.NewPublisher(logger, runtimeDeps.JS, cfg.SubjectOut) matcher := detect.NewHTTPBodyFingerprintMatcher(5 * time.Second) - classifier := detect.NewClassifier(matcher).WithLogger(logger) - worker := app.NewWorker(logger, pub.Publish, classifier.Classify) + classifier := detect.NewClassifier(matcher, logger) + worker := app.NewWorker(logger, pub, classifier) go func() { <-sig diff --git a/scanners/subdomain-takeover/internal/app/runner.go b/scanners/subdomain-takeover/internal/app/runner.go index 2d3547373b..0e0d6f7f9f 100644 --- a/scanners/subdomain-takeover/internal/app/runner.go +++ b/scanners/subdomain-takeover/internal/app/runner.go @@ -20,8 +20,6 @@ type RunnerDeps struct { NC *nats.Conn } -var checkConnection = messaging.CheckConnection - func isBenignNextError(err error) bool { return errors.Is(err, jetstream.ErrMsgIteratorClosed) || errors.Is(err, jetstream.ErrNoMessages) || @@ -45,7 +43,7 @@ func Run(ctx context.Context, deps RunnerDeps) { Loop: for { - if err := checkConnection(deps.NC); err != nil { + if err := messaging.CheckConnection(deps.NC); err != nil { logger.Error().Err(err).Msg("NATS connection unhealthy") break Loop } diff --git a/scanners/subdomain-takeover/internal/app/runner_test.go b/scanners/subdomain-takeover/internal/app/runner_test.go deleted file mode 100644 index be061d6bae..0000000000 --- a/scanners/subdomain-takeover/internal/app/runner_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package app - -import ( - "context" - "errors" - "sync" - "testing" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - "github.com/rs/zerolog" -) - -type fakeMessagesIter struct { - mu sync.Mutex - msgs []jetstream.Msg - err error - idx int - stopCalled bool -} - -func (f *fakeMessagesIter) Next(...jetstream.NextOpt) (jetstream.Msg, error) { - f.mu.Lock() - defer f.mu.Unlock() - - if f.err != nil { - return nil, f.err - } - if f.idx >= len(f.msgs) { - return nil, jetstream.ErrMsgIteratorClosed - } - m := f.msgs[f.idx] - f.idx++ - return m, nil -} - -func (f *fakeMessagesIter) Stop() { f.stopCalled = true } -func (f *fakeMessagesIter) Drain() {} - -// nolint:revive // nats Conn fields are unexported; this fake nil connection path only. -func TestRun_ReturnsImmediatelyWhenConnectionUnhealthy(t *testing.T) { - iter := &fakeMessagesIter{} - worker := NewWorker(zerolog.Nop(), (&fakePublisher{}).Publish, (fakeClassifier{}).Classify) - - deps := RunnerDeps{ - Logger: zerolog.Nop(), - WorkerCount: 2, - Iter: iter, - Worker: worker, - NC: nil, - } - - Run(context.Background(), deps) - - if iter.idx != 0 { - t.Fatalf("expected no iterator reads, got %d", iter.idx) - } -} - -func TestRun_ClampsWorkerCountBelowOne(t *testing.T) { - origCheckConnection := checkConnection - t.Cleanup(func() { checkConnection = origCheckConnection }) - - checkCalls := 0 - checkConnection = func(_ *nats.Conn) error { - checkCalls++ - if checkCalls > 1 { - return errors.New("stop") - } - return nil - } - - msg := &fakeJSMsg{data: []byte(`{"domain_key":"k","results":{}}`), subject: "scans.dns_scanner_results"} - iter := &fakeMessagesIter{msgs: []jetstream.Msg{msg}} - worker := NewWorker(zerolog.Nop(), (&fakePublisher{}).Publish, (fakeClassifier{}).Classify) - - deps := RunnerDeps{ - Logger: zerolog.Nop(), - WorkerCount: 0, - Iter: iter, - Worker: worker, - NC: nil, - } - - Run(context.Background(), deps) - if msg.ackCount != 1 { - t.Fatalf("expected one handled message (ack once), got ack=%d", msg.ackCount) - } -} - -func TestRun_ExitsWhenContextCancelledDuringNextErrors(t *testing.T) { - origCheckConnection := checkConnection - t.Cleanup(func() { checkConnection = origCheckConnection }) - checkConnection = func(_ *nats.Conn) error { return nil } - - iter := &fakeMessagesIter{err: errors.New("next failed")} - worker := NewWorker(zerolog.Nop(), (&fakePublisher{}).Publish, (fakeClassifier{}).Classify) - - ctx, cancel := context.WithCancel(context.Background()) - go func() { - time.Sleep(5 * time.Millisecond) - cancel() - }() - - deps := RunnerDeps{ - Logger: zerolog.Nop(), - WorkerCount: 1, - Iter: iter, - Worker: worker, - NC: nil, - } - - Run(ctx, deps) - - if iter.idx != 0 { - t.Fatalf("expected no iterator reads, got %d", iter.idx) - } -} diff --git a/scanners/subdomain-takeover/internal/app/worker.go b/scanners/subdomain-takeover/internal/app/worker.go index 79b59fa57e..7c71414011 100644 --- a/scanners/subdomain-takeover/internal/app/worker.go +++ b/scanners/subdomain-takeover/internal/app/worker.go @@ -6,26 +6,24 @@ import ( "encoding/json" "fmt" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" "github.com/nats-io/nats.go/jetstream" "github.com/rs/zerolog" ) type Worker struct { - logger zerolog.Logger - publishFinding func(ctx context.Context, finding model.Finding) error - classifyFinding func(input model.Input) ([]model.Finding, error) + logger zerolog.Logger + publisher *messaging.Publisher + classifier *detect.Classifier } -func NewWorker( - logger zerolog.Logger, - publishFinding func(ctx context.Context, finding model.Finding) error, - classifyFinding func(input model.Input) ([]model.Finding, error), -) *Worker { +func NewWorker(logger zerolog.Logger, publisher *messaging.Publisher, classifier *detect.Classifier) *Worker { return &Worker{ - logger: logger, - publishFinding: publishFinding, - classifyFinding: classifyFinding, + logger: logger, + publisher: publisher, + classifier: classifier, } } @@ -47,7 +45,7 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { log = log.With().Str("domain_key", scan.DomainKey).Logger() - findings, err := w.classifyFinding(scan) + findings, err := w.classifier.Classify(scan) if err != nil { log.Err(err).Msg("classify error") if nakErr := msg.Nak(); nakErr != nil { @@ -61,7 +59,7 @@ func (w *Worker) Handle(ctx context.Context, msg jetstream.Msg) error { } for _, finding := range findings { - err = w.publishFinding(ctx, finding) + err = w.publisher.Publish(ctx, finding) if err != nil { log.Err(err).Msg("publish error") if nakErr := msg.Nak(); nakErr != nil { diff --git a/scanners/subdomain-takeover/internal/app/worker_test.go b/scanners/subdomain-takeover/internal/app/worker_test.go deleted file mode 100644 index 8a41806b2c..0000000000 --- a/scanners/subdomain-takeover/internal/app/worker_test.go +++ /dev/null @@ -1,196 +0,0 @@ -package app - -import ( - "context" - "errors" - "testing" - "time" - - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - "github.com/rs/zerolog" -) - -type fakePublisher struct { - err error - published []model.Finding -} - -func (f *fakePublisher) Publish(_ context.Context, finding model.Finding) error { - if f.err != nil { - return f.err - } - f.published = append(f.published, finding) - return nil -} - -type fakeClassifier struct { - findings []model.Finding - err error -} - -func (f fakeClassifier) Classify(model.Input) ([]model.Finding, error) { - if f.err != nil { - return nil, f.err - } - return f.findings, nil -} - -type fakeJSMsg struct { - data []byte - subject string - - ackErr error - nakErr error - termErr error - - ackCount int - nakCount int - termCount int -} - -func (m *fakeJSMsg) Metadata() (*jetstream.MsgMetadata, error) { return nil, nil } -func (m *fakeJSMsg) Data() []byte { return m.data } -func (m *fakeJSMsg) Headers() nats.Header { return nil } -func (m *fakeJSMsg) Subject() string { return m.subject } -func (m *fakeJSMsg) Reply() string { return "" } - -func (m *fakeJSMsg) Ack() error { - m.ackCount++ - return m.ackErr -} - -func (m *fakeJSMsg) DoubleAck(context.Context) error { return nil } - -func (m *fakeJSMsg) Nak() error { - m.nakCount++ - return m.nakErr -} - -func (m *fakeJSMsg) NakWithDelay(time.Duration) error { return nil } -func (m *fakeJSMsg) InProgress() error { return nil } - -func (m *fakeJSMsg) Term() error { - m.termCount++ - return m.termErr -} - -func (m *fakeJSMsg) TermWithReason(string) error { return nil } - -func TestWorkerHandle(t *testing.T) { - logger := zerolog.Nop() - - tests := []struct { - name string - data []byte - ackErr error - nakErr error - termErr error - classifier fakeClassifier - publisherErr error - wantErr bool - wantAck int - wantNak int - wantTerm int - wantPublishedFindings int - }{ - { - name: "decode error terminates message", - data: []byte("{not-json"), - wantErr: true, - wantAck: 0, - wantNak: 0, - wantTerm: 1, - }, - { - name: "classification error naks message", - data: []byte(`{"domain_key":"k","results":{}}`), - classifier: fakeClassifier{err: errors.New("classify failed")}, - wantErr: true, - wantAck: 0, - wantNak: 1, - wantTerm: 0, - }, - { - name: "publish error naks message", - data: []byte(`{"domain_key":"k","results":{}}`), - classifier: fakeClassifier{findings: []model.Finding{{Domain: "a.example.ca"}}}, - publisherErr: errors.New("publish failed"), - wantErr: true, - wantAck: 0, - wantNak: 1, - wantTerm: 0, - }, - { - name: "successful processing publishes all findings and acks", - data: []byte(`{"domain_key":"k","results":{}}`), - classifier: fakeClassifier{findings: []model.Finding{{Domain: "a.example.ca"}, {Domain: "b.example.ca"}}}, - wantErr: false, - wantAck: 1, - wantNak: 0, - wantTerm: 0, - wantPublishedFindings: 2, - }, - { - name: "ack failure returns error", - data: []byte(`{"domain_key":"k","results":{}}`), - ackErr: errors.New("ack failed"), - wantErr: true, - wantAck: 1, - wantNak: 0, - wantTerm: 0, - }, - { - name: "decode error still returned when term fails", - data: []byte("{bad-json"), - termErr: errors.New("term failed"), - wantErr: true, - wantAck: 0, - wantNak: 0, - wantTerm: 1, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - pub := &fakePublisher{err: tc.publisherErr} - worker := NewWorker(logger, pub.Publish, tc.classifier.Classify) - - msg := &fakeJSMsg{ - data: tc.data, - subject: "scans.dns_scanner_results", - ackErr: tc.ackErr, - nakErr: tc.nakErr, - termErr: tc.termErr, - } - - err := worker.Handle(context.Background(), msg) - if tc.wantErr && err == nil { - t.Fatal("expected error") - } - if !tc.wantErr && err != nil { - t.Fatalf("unexpected error: %v", err) - } - if msg.ackCount != tc.wantAck { - t.Fatalf("unexpected ack count: got=%d want=%d", msg.ackCount, tc.wantAck) - } - if msg.nakCount != tc.wantNak { - t.Fatalf("unexpected nak count: got=%d want=%d", msg.nakCount, tc.wantNak) - } - if msg.termCount != tc.wantTerm { - t.Fatalf("unexpected term count: got=%d want=%d", msg.termCount, tc.wantTerm) - } - if len(pub.published) != tc.wantPublishedFindings { - t.Fatalf("unexpected published findings count: got=%d want=%d", len(pub.published), tc.wantPublishedFindings) - } - }) - } -} - -func TestDecodeScan_TrimsTrailingNewline(t *testing.T) { - _, err := decodeScan([]byte("{\"domain_key\":\"k\",\"results\":{}}\n")) - if err != nil { - t.Fatalf("expected decode success, got error: %v", err) - } -} diff --git a/scanners/subdomain-takeover/internal/detect/classifier.go b/scanners/subdomain-takeover/internal/detect/classifier.go index 45289b388e..0f7c26984a 100644 --- a/scanners/subdomain-takeover/internal/detect/classifier.go +++ b/scanners/subdomain-takeover/internal/detect/classifier.go @@ -7,35 +7,21 @@ import ( ) type Classifier struct { - Matcher BodyFingerprintMatcher + matcher BodyFingerprintMatcher logger zerolog.Logger } -func NewClassifier(matcher BodyFingerprintMatcher) *Classifier { - return &Classifier{Matcher: matcher, logger: zerolog.Nop()} -} - -func (c *Classifier) WithLogger(logger zerolog.Logger) *Classifier { - c.logger = logger.With().Str("component", "classifier").Logger() - SetLogger(logger) - return c -} - -func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { - matcher := c.Matcher +func NewClassifier(matcher BodyFingerprintMatcher, logger zerolog.Logger) *Classifier { if matcher == nil { matcher = NewNoopBodyFingerprintMatcher() } - - logger := c.logger - if logger.GetLevel() == zerolog.NoLevel { - logger = zerolog.Nop() + return &Classifier{ + matcher: matcher, + logger: logger.With().Str("component", "classifier").Logger(), } - - return Classify(input, matcher, logger) } -func Classify(input model.Input, matcher BodyFingerprintMatcher, logger zerolog.Logger) ([]model.Finding, error) { +func (c *Classifier) Classify(input model.Input) ([]model.Finding, error) { findings := []model.Finding{} cnameProviderFingerprints := fingerprints.CNAME() @@ -43,10 +29,10 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, logger zerolog. cnameEvidence := ExtractCNAMEEvidence(input.Results) if cnameEvidence != nil { - logger.Debug().Str("domain_key", input.DomainKey).Str("domain", cnameEvidence.Domain).Msg("cname evidence extracted") - cnameHit := MatchCNAMEFingerprints(*cnameEvidence, cnameProviderFingerprints, matcher) + c.logger.Debug().Str("domain_key", input.DomainKey).Str("domain", cnameEvidence.Domain).Msg("cname evidence extracted") + cnameHit := MatchCNAMEFingerprints(*cnameEvidence, cnameProviderFingerprints, c.matcher, c.logger) if ShouldEmitCNAME(cnameHit) { - logger.Debug(). + c.logger.Debug(). Str("domain_key", input.DomainKey). Str("domain", cnameEvidence.Domain). Str("provider", cnameHit.Provider). @@ -61,9 +47,8 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, logger zerolog. ReasonCode: string(cnameHit.ReasonCode), Confidence: ConfidenceForReason(cnameHit.ReasonCode), }) - } - if cnameHit != nil && !ShouldEmitCNAME(cnameHit) { - logger.Debug(). + } else if cnameHit != nil { + c.logger.Debug(). Str("domain_key", input.DomainKey). Str("domain", cnameEvidence.Domain). Str("provider", cnameHit.Provider). @@ -71,19 +56,19 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, logger zerolog. Msg("cname finding suppressed") } } else { - logger.Debug().Str("domain_key", input.DomainKey).Msg("no cname evidence") + c.logger.Debug().Str("domain_key", input.DomainKey).Msg("no cname evidence") } nsEvidence := ExtractNSEvidence(input.Results) if nsEvidence != nil { - logger.Debug(). + c.logger.Debug(). Str("domain_key", input.DomainKey). Str("domain", nsEvidence.Domain). Int("ns_hosts", len(nsEvidence.NSHosts)). Msg("ns evidence extracted") - nsHit := MatchNSProviderRules(*nsEvidence, nsProviderFingerprints) + nsHit := MatchNSProviderRules(*nsEvidence, nsProviderFingerprints, c.logger) if ShouldEmitNSHijack(nsHit) { - logger.Debug(). + c.logger.Debug(). Str("domain_key", input.DomainKey). Str("domain", nsEvidence.Domain). Str("provider", nsHit.Provider). @@ -100,7 +85,7 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, logger zerolog. Confidence: ConfidenceForReason(nsHit.ReasonCode), }) } else if nsHit != nil { - logger.Debug(). + c.logger.Debug(). Str("domain_key", input.DomainKey). Str("domain", nsEvidence.Domain). Str("provider", nsHit.Provider). @@ -109,7 +94,7 @@ func Classify(input model.Input, matcher BodyFingerprintMatcher, logger zerolog. Msg("ns finding suppressed") } } else { - logger.Debug().Str("domain_key", input.DomainKey).Msg("no ns evidence") + c.logger.Debug().Str("domain_key", input.DomainKey).Msg("no ns evidence") } return findings, nil diff --git a/scanners/subdomain-takeover/internal/detect/classifier_test.go b/scanners/subdomain-takeover/internal/detect/classifier_test.go deleted file mode 100644 index 49191242f5..0000000000 --- a/scanners/subdomain-takeover/internal/detect/classifier_test.go +++ /dev/null @@ -1,139 +0,0 @@ -package detect - -import ( - "testing" - - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" - "github.com/rs/zerolog" -) - -func TestClassify_ExpectedBehavior(t *testing.T) { - if err := fingerprints.Load(zerolog.Nop()); err != nil { - t.Fatalf("failed to load fingerprints: %v", err) - } - - t.Run("emits cname and ns findings when both are exploitable", func(t *testing.T) { - rdapMatch := true - input := model.Input{ - DomainKey: "k1", - Results: model.ScanResults{ - Domain: strPtr("a.example.ca"), - CnameRecord: strPtr("a.example.ca. 300 IN CNAME foo.azurewebsites.net."), - RegistrarContext: &model.RegistrarContext{ - LookupSuccess: true, - RegistrarName: "Namecheap", - DelegationMatchesRDAP: &rdapMatch, - }, - NsDelegations: &model.NsDelegations{ - Hosts: []string{"ns1.digitalocean.com"}, - Delegation: model.Delegation{ - LameType: "partial", - }, - }, - }, - } - - findings, err := Classify(input, fakeMatcher{}, zerolog.Nop()) - if err != nil { - t.Fatalf("Classify error: %v", err) - } - if len(findings) != 2 { - t.Fatalf("expected 2 findings, got %d", len(findings)) - } - - var sawCNAME, sawNS bool - for _, f := range findings { - if f.RecordType == model.RecordTypeCNAME { - sawCNAME = true - if f.ReasonCode != string(ReasonCNAMEDanglingNXDOMAIN) { - t.Fatalf("unexpected cname reason: %q", f.ReasonCode) - } - if f.Confidence != ConfidenceProbable { - t.Fatalf("unexpected cname confidence: %q", f.Confidence) - } - } - if f.RecordType == model.RecordTypeNS { - sawNS = true - if f.ReasonCode != string(ReasonNSPartialLameProviderVulnerable) { - t.Fatalf("unexpected ns reason: %q", f.ReasonCode) - } - if f.Confidence != ConfidenceProbable { - t.Fatalf("unexpected ns confidence: %q", f.Confidence) - } - } - } - if !sawCNAME || !sawNS { - t.Fatalf("missing expected finding types: cname=%v ns=%v", sawCNAME, sawNS) - } - }) - - t.Run("suppresses non-exploitable ns matches", func(t *testing.T) { - input := model.Input{ - DomainKey: "k2", - Results: model.ScanResults{ - Domain: strPtr("b.example.ca"), - CnameRecord: strPtr("b.example.ca. 300 IN CNAME foo.ghost.io."), - NsDelegations: &model.NsDelegations{ - Hosts: []string{"aria.ns.cloudflare.com"}, - Delegation: model.Delegation{ - LameType: "full", - }, - }, - }, - } - - matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { - return true - }} - - findings, err := Classify(input, matcher, zerolog.Nop()) - if err != nil { - t.Fatalf("Classify error: %v", err) - } - - if len(findings) != 1 { - t.Fatalf("expected 1 finding, got %d", len(findings)) - } - if findings[0].RecordType != model.RecordTypeCNAME { - t.Fatalf("expected cname-only finding, got %s", findings[0].RecordType) - } - if findings[0].ReasonCode != string(ReasonCNAMEProviderFingerprintBodyMatch) { - t.Fatalf("unexpected cname reason: %q", findings[0].ReasonCode) - } - if findings[0].Confidence != ConfidenceProbable { - t.Fatalf("unexpected cname confidence: %q", findings[0].Confidence) - } - }) - - t.Run("returns no findings and no panic when evidence absent", func(t *testing.T) { - input := model.Input{DomainKey: "k3", Results: model.ScanResults{}} - findings, err := Classify(input, fakeMatcher{}, zerolog.Nop()) - if err != nil { - t.Fatalf("Classify error: %v", err) - } - if len(findings) != 0 { - t.Fatalf("expected no findings, got %d", len(findings)) - } - }) -} - -func TestClassifier_MethodDefaults(t *testing.T) { - classifier := NewClassifier(nil) - if classifier == nil { - t.Fatal("expected classifier") - } - - input := model.Input{DomainKey: "k", Results: model.ScanResults{}} - findings, err := classifier.Classify(input) - if err != nil { - t.Fatalf("Classify error: %v", err) - } - if len(findings) != 0 { - t.Fatalf("expected no findings, got %d", len(findings)) - } - - if classifier.WithLogger(zerolog.Nop()) != classifier { - t.Fatal("WithLogger should return same classifier pointer") - } -} diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules.go b/scanners/subdomain-takeover/internal/detect/cname_rules.go index bc1023182e..f69f862aa1 100644 --- a/scanners/subdomain-takeover/internal/detect/cname_rules.go +++ b/scanners/subdomain-takeover/internal/detect/cname_rules.go @@ -1,6 +1,9 @@ package detect -import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" +import ( + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/rs/zerolog" +) type CNAMEHit struct { Matched bool @@ -9,10 +12,10 @@ type CNAMEHit struct { NeedsNX bool } -func MatchCNAMEFingerprints(evidence CNAMEEvidence, providerFingerprints []fingerprints.CNAMEProviderFingerprint, matcher BodyFingerprintMatcher) *CNAMEHit { +func MatchCNAMEFingerprints(evidence CNAMEEvidence, providerFingerprints []fingerprints.CNAMEProviderFingerprint, matcher BodyFingerprintMatcher, logger zerolog.Logger) *CNAMEHit { for _, fp := range providerFingerprints { if fp.ContainsTarget(evidence.Target) { - detectLogger.Debug(). + logger.Debug(). Str("domain", evidence.Domain). Str("target", evidence.Target). Str("provider", fp.Name). @@ -35,7 +38,7 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, providerFingerprints []finge } else { hit.ReasonCode = ReasonCNAMETargetMatchMissingBodyFP mode := fingerprints.NormalizeMode(fp.Mode, fp.Fingerprint) - detectLogger.Debug(). + logger.Debug(). Str("domain", evidence.Domain). Str("provider", fp.Name). Str("fingerprint_mode", string(mode)). @@ -43,13 +46,13 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, providerFingerprints []finge if matcher != nil && matcher.Contains(evidence.Domain, fp.Fingerprint, mode) { hit.Matched = true hit.ReasonCode = ReasonCNAMEProviderFingerprintBodyMatch - detectLogger.Debug(). + logger.Debug(). Str("domain", evidence.Domain). Str("provider", fp.Name). Str("reason_code", string(hit.ReasonCode)). Msg("cname body fingerprint matched") } else { - detectLogger.Debug(). + logger.Debug(). Str("domain", evidence.Domain). Str("provider", fp.Name). Str("reason_code", string(hit.ReasonCode)). @@ -58,7 +61,7 @@ func MatchCNAMEFingerprints(evidence CNAMEEvidence, providerFingerprints []finge } if hit.NeedsNX && !hit.Matched { - detectLogger.Debug(). + logger.Debug(). Str("domain", evidence.Domain). Str("provider", fp.Name). Str("reason_code", string(hit.ReasonCode)). diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules_test.go b/scanners/subdomain-takeover/internal/detect/cname_rules_test.go deleted file mode 100644 index bb45a7338b..0000000000 --- a/scanners/subdomain-takeover/internal/detect/cname_rules_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package detect - -import ( - "testing" - - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" -) - -func TestMatchCNAMEFingerprints(t *testing.T) { - fps := []fingerprints.CNAMEProviderFingerprint{ - { - Name: "Azure", - Cname: []string{"azurewebsites.net"}, - Nxdomain: true, - Fingerprint: "This web app is stopped", - }, - { - Name: "Ghost", - Cname: []string{"ghost.io"}, - Nxdomain: false, - Fingerprint: "The thing you were looking for is no longer here", - Mode: fingerprints.FingerprintModeLiteral, - }, - } - - t.Run("returns nil when no target match", func(t *testing.T) { - evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "foo.not-a-provider.net", NoResolve: true} - if got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{}); got != nil { - t.Fatalf("expected nil, got %+v", got) - } - }) - - t.Run("nxdomain provider emits dangling reason when unresolved", func(t *testing.T) { - evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "foo.azurewebsites.net", NoResolve: true} - got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{}) - if got == nil { - t.Fatal("expected hit, got nil") - } - if !got.Matched { - t.Fatal("expected matched=true") - } - if got.ReasonCode != ReasonCNAMEDanglingNXDOMAIN { - t.Fatalf("unexpected reason: %q", got.ReasonCode) - } - }) - - t.Run("nxdomain provider suppressed when resolve evidence exists", func(t *testing.T) { - evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "foo.azurewebsites.net", NoResolve: false} - got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{}) - if got == nil { - t.Fatal("expected hit, got nil") - } - if got.Matched { - t.Fatal("expected matched=false") - } - if got.ReasonCode != ReasonCNAMETargetMatchMissingNXDOMAIN { - t.Fatalf("unexpected reason: %q", got.ReasonCode) - } - }) - - t.Run("body fingerprint provider emits when matcher returns true", func(t *testing.T) { - evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "blog.ghost.io", NoResolve: false} - matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { - if domain != "a.example.ca" || mode != fingerprints.FingerprintModeLiteral { - t.Fatalf("unexpected matcher args: domain=%q mode=%q", domain, mode) - } - return true - }} - got := MatchCNAMEFingerprints(evidence, fps, matcher) - if got == nil { - t.Fatal("expected hit, got nil") - } - if !got.Matched { - t.Fatal("expected matched=true") - } - if got.ReasonCode != ReasonCNAMEProviderFingerprintBodyMatch { - t.Fatalf("unexpected reason: %q", got.ReasonCode) - } - }) - - t.Run("body fingerprint provider suppressed when matcher false", func(t *testing.T) { - evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "blog.ghost.io", NoResolve: false} - got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{containsFn: func(string, string, fingerprints.FingerprintMode) bool { return false }}) - if got == nil { - t.Fatal("expected hit, got nil") - } - if got.Matched { - t.Fatal("expected matched=false") - } - if got.ReasonCode != ReasonCNAMETargetMatchMissingBodyFP { - t.Fatalf("unexpected reason: %q", got.ReasonCode) - } - }) -} - -func TestShouldEmitCNAME(t *testing.T) { - tests := []struct { - name string - hit *CNAMEHit - want bool - }{ - {name: "nil hit", hit: nil, want: false}, - {name: "unmatched hit", hit: &CNAMEHit{Matched: false}, want: false}, - {name: "matched hit", hit: &CNAMEHit{Matched: true}, want: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := ShouldEmitCNAME(tt.hit); got != tt.want { - t.Fatalf("ShouldEmitCNAME()=%v want=%v", got, tt.want) - } - }) - } -} diff --git a/scanners/subdomain-takeover/internal/detect/logger.go b/scanners/subdomain-takeover/internal/detect/logger.go deleted file mode 100644 index c3d4a30293..0000000000 --- a/scanners/subdomain-takeover/internal/detect/logger.go +++ /dev/null @@ -1,9 +0,0 @@ -package detect - -import "github.com/rs/zerolog" - -var detectLogger = zerolog.Nop() - -func SetLogger(logger zerolog.Logger) { - detectLogger = logger.With().Str("component", "detect").Logger() -} diff --git a/scanners/subdomain-takeover/internal/detect/logger_test.go b/scanners/subdomain-takeover/internal/detect/logger_test.go deleted file mode 100644 index 2487426271..0000000000 --- a/scanners/subdomain-takeover/internal/detect/logger_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package detect - -import ( - "io" - "testing" - - "github.com/rs/zerolog" -) - -func TestSetLogger(t *testing.T) { - SetLogger(zerolog.New(io.Discard)) - if got := detectLogger.GetLevel(); got == zerolog.Disabled { - t.Fatalf("expected configured logger, got level=%s", got) - } -} diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go index bc0a672aca..eacc7d6b30 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_reasoning.go +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning.go @@ -58,10 +58,6 @@ func getRegistrarReasonCode(provider string, registrar *model.RegistrarContext) return "" } -func isRegistrarMismatch(provider string, registrar *model.RegistrarContext) bool { - return getRegistrarReasonCode(provider, registrar) == "" -} - func isExploitableProviderStatus(status fingerprints.NSProviderStatus) bool { return status == fingerprints.NSStatusVulnerable || status == fingerprints.NSStatusVulnerableWithPurchase } diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go deleted file mode 100644 index 742e510002..0000000000 --- a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go +++ /dev/null @@ -1,167 +0,0 @@ -package detect - -import ( - "testing" - - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" -) - -func TestGetNSHijackReasonCode(t *testing.T) { - namecheapRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"} - digitalOceanRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."} - invalidRegistrar := &model.RegistrarContext{LookupSuccess: false, RegistrarName: "Namecheap"} - - tests := []struct { - name string - lameType string - provider fingerprints.NSProviderFingerprint - registrar *model.RegistrarContext - want ReasonCode - }{ - { - name: "full vulnerable with registrar mismatch", - lameType: "full", - provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, - registrar: namecheapRegistrar, - want: ReasonNSFullLameProviderVulnerable, - }, - { - name: "partial vulnerable with registrar mismatch", - lameType: "partial", - provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, - registrar: namecheapRegistrar, - want: ReasonNSPartialLameProviderVulnerable, - }, - { - name: "full vulnerable with purchase and mismatch", - lameType: "full", - provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerableWithPurchase}, - registrar: namecheapRegistrar, - want: ReasonNSFullLameProviderVulnerable, - }, - { - name: "full non vulnerable", - lameType: "full", - provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusNotVulnerable}, - registrar: namecheapRegistrar, - want: ReasonNSLameProviderUnknown, - }, - { - name: "unknown lame type", - lameType: "none", - provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, - registrar: namecheapRegistrar, - want: ReasonNSProviderMatchOnly, - }, - { - name: "same registrar/provider suppressed", - lameType: "full", - provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, - registrar: digitalOceanRegistrar, - want: ReasonNSRegistrarProviderMatch, - }, - { - name: "missing registrar context suppressed", - lameType: "full", - provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, - registrar: nil, - want: ReasonNSRegistrarContextInsufficient, - }, - { - name: "invalid registrar context suppressed", - lameType: "full", - provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, - registrar: invalidRegistrar, - want: ReasonNSRegistrarContextInsufficient, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := getNSHijackReasonCode(tt.lameType, tt.provider, tt.registrar) - if got != tt.want { - t.Fatalf("getNSHijackReasonCode()=%q want=%q", got, tt.want) - } - }) - } -} - -func TestNSReasoningHelpers(t *testing.T) { - t.Run("isExploitableProviderStatus", func(t *testing.T) { - tests := []struct { - name string - status fingerprints.NSProviderStatus - want bool - }{ - {name: "vulnerable", status: fingerprints.NSStatusVulnerable, want: true}, - {name: "vulnerable with purchase", status: fingerprints.NSStatusVulnerableWithPurchase, want: true}, - {name: "not vulnerable", status: fingerprints.NSStatusNotVulnerable, want: false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := isExploitableProviderStatus(tt.status); got != tt.want { - t.Fatalf("isExploitableProviderStatus(%q)=%v want=%v", tt.status, got, tt.want) - } - }) - } - }) - - t.Run("normalizeLameType", func(t *testing.T) { - if got := normalizeLameType(" Partial "); got != "partial" { - t.Fatalf("normalizeLameType mismatch: %q", got) - } - }) - - t.Run("isRegistrarMismatch", func(t *testing.T) { - tests := []struct { - name string - provider string - registrar *model.RegistrarContext - want bool - }{ - { - name: "different providers", - provider: "Digital Ocean", - registrar: &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"}, - want: true, - }, - { - name: "same provider", - provider: "Digital Ocean", - registrar: &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."}, - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := isRegistrarMismatch(tt.provider, tt.registrar); got != tt.want { - t.Fatalf("isRegistrarMismatch()=%v want=%v", got, tt.want) - } - }) - } - }) - - t.Run("nsReasonRank ordering", func(t *testing.T) { - checks := []struct { - name string - high ReasonCode - low ReasonCode - label string - }{ - {name: "full above partial", high: ReasonNSFullLameProviderVulnerable, low: ReasonNSPartialLameProviderVulnerable, label: "full lame > partial lame"}, - {name: "partial above registrar-provider-match", high: ReasonNSPartialLameProviderVulnerable, low: ReasonNSRegistrarProviderMatch, label: "partial lame > registrar-provider-match"}, - {name: "registrar gate above provider only", high: ReasonNSRegistrarProviderMatch, low: ReasonNSProviderMatchOnly, label: "registrar gate > provider-only"}, - } - - for _, check := range checks { - t.Run(check.name, func(t *testing.T) { - if nsReasonRank(check.high) <= nsReasonRank(check.low) { - t.Fatalf("expected %s", check.label) - } - }) - } - }) -} diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules.go b/scanners/subdomain-takeover/internal/detect/ns_rules.go index 32de3049e3..49b7dabcf3 100644 --- a/scanners/subdomain-takeover/internal/detect/ns_rules.go +++ b/scanners/subdomain-takeover/internal/detect/ns_rules.go @@ -1,6 +1,9 @@ package detect -import "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" +import ( + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/rs/zerolog" +) type NSHit struct { Matched bool @@ -9,9 +12,9 @@ type NSHit struct { ReasonCode ReasonCode } -func MatchNSProviderRules(evidence NSEvidence, providerFingerprints []fingerprints.NSProviderFingerprint) *NSHit { +func MatchNSProviderRules(evidence NSEvidence, providerFingerprints []fingerprints.NSProviderFingerprint, logger zerolog.Logger) *NSHit { if len(evidence.NSHosts) == 0 || len(providerFingerprints) == 0 { - detectLogger.Debug().Int("ns_hosts", len(evidence.NSHosts)).Int("fingerprints", len(providerFingerprints)).Msg("skipping ns matching due to insufficient inputs") + logger.Debug().Int("ns_hosts", len(evidence.NSHosts)).Int("fingerprints", len(providerFingerprints)).Msg("skipping ns matching due to insufficient inputs") return nil } @@ -32,7 +35,7 @@ func MatchNSProviderRules(evidence NSEvidence, providerFingerprints []fingerprin rank := nsReasonRank(hit.ReasonCode) - detectLogger.Debug(). + logger.Debug(). Str("domain", evidence.Domain). Str("host", host). Str("provider", fp.Name). @@ -51,11 +54,11 @@ func MatchNSProviderRules(evidence NSEvidence, providerFingerprints []fingerprin } if best == nil { - detectLogger.Debug().Str("domain", evidence.Domain).Msg("no ns provider match") + logger.Debug().Str("domain", evidence.Domain).Msg("no ns provider match") return nil } - detectLogger.Debug(). + logger.Debug(). Str("domain", evidence.Domain). Str("host", best.Host). Str("provider", best.Provider). diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go deleted file mode 100644 index 15167d8622..0000000000 --- a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go +++ /dev/null @@ -1,168 +0,0 @@ -package detect - -import ( - "testing" - - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" -) - -func TestMatchNSProviderRules(t *testing.T) { - nsFP := []fingerprints.NSProviderFingerprint{ - {Name: "UnknownDNS", Status: fingerprints.NSStatusNotVulnerable, HostPatterns: []string{"*.unknown-dns.net"}}, - {Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, - } - - namecheapRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"} - digitalOceanRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."} - - t.Run("returns nil for missing hosts", func(t *testing.T) { - evidence := NSEvidence{Domain: "a.example.ca"} - if got := MatchNSProviderRules(evidence, nsFP); got != nil { - t.Fatalf("expected nil, got %+v", got) - } - }) - - t.Run("returns nil for missing fingerprints", func(t *testing.T) { - evidence := NSEvidence{Domain: "a.example.ca", NSHosts: []string{"ns1.risky-dns.net"}} - if got := MatchNSProviderRules(evidence, nil); got != nil { - t.Fatalf("expected nil, got %+v", got) - } - }) - - t.Run("returns nil when no provider matches", func(t *testing.T) { - evidence := NSEvidence{Domain: "a.example.ca", NSHosts: []string{"ns1.nomatch.net"}} - if got := MatchNSProviderRules(evidence, nsFP); got != nil { - t.Fatalf("expected nil, got %+v", got) - } - }) - - t.Run("matches vulnerable full lame and emits correct reason", func(t *testing.T) { - evidence := NSEvidence{ - Domain: "a.example.ca", - NSHosts: []string{"ns1.risky-dns.net"}, - Registrar: namecheapRegistrar, - NSDelegations: model.NsDelegations{Delegation: model.Delegation{ - LameType: "full", - }}, - } - got := MatchNSProviderRules(evidence, nsFP) - if got == nil { - t.Fatal("expected hit, got nil") - } - if got.Provider != "Digital Ocean" { - t.Fatalf("unexpected provider: %q", got.Provider) - } - if got.ReasonCode != ReasonNSFullLameProviderVulnerable { - t.Fatalf("unexpected reason: %q", got.ReasonCode) - } - if !got.Matched { - t.Fatal("expected matched=true") - } - }) - - t.Run("ranks vulnerable higher than unknown", func(t *testing.T) { - evidence := NSEvidence{ - Domain: "a.example.ca", - NSHosts: []string{"ns1.unknown-dns.net", "ns1.risky-dns.net"}, - Registrar: namecheapRegistrar, - NSDelegations: model.NsDelegations{Delegation: model.Delegation{ - LameType: "partial", - }}, - } - got := MatchNSProviderRules(evidence, nsFP) - if got == nil { - t.Fatal("expected hit, got nil") - } - if got.Provider != "Digital Ocean" { - t.Fatalf("expected risky provider to win ranking, got %q", got.Provider) - } - if got.ReasonCode != ReasonNSPartialLameProviderVulnerable { - t.Fatalf("unexpected reason: %q", got.ReasonCode) - } - }) - - t.Run("non-vulnerable full lame is classified but not emittable", func(t *testing.T) { - evidence := NSEvidence{ - Domain: "a.example.ca", - NSHosts: []string{"ns1.unknown-dns.net"}, - Registrar: namecheapRegistrar, - NSDelegations: model.NsDelegations{Delegation: model.Delegation{ - LameType: "full", - }}, - } - got := MatchNSProviderRules(evidence, nsFP) - if got == nil { - t.Fatal("expected hit, got nil") - } - if got.ReasonCode != ReasonNSLameProviderUnknown { - t.Fatalf("unexpected reason: %q", got.ReasonCode) - } - if got.Matched { - t.Fatal("expected matched=false for non-vulnerable state") - } - }) - - t.Run("suppresses vulnerable provider when registrar matches provider", func(t *testing.T) { - evidence := NSEvidence{ - Domain: "a.example.ca", - NSHosts: []string{"ns1.risky-dns.net"}, - Registrar: digitalOceanRegistrar, - NSDelegations: model.NsDelegations{Delegation: model.Delegation{ - LameType: "full", - }}, - } - - got := MatchNSProviderRules(evidence, nsFP) - if got == nil { - t.Fatal("expected hit, got nil") - } - if got.ReasonCode != ReasonNSRegistrarProviderMatch { - t.Fatalf("unexpected reason: %q", got.ReasonCode) - } - if got.Matched { - t.Fatal("expected matched=false when registrar and provider match") - } - }) - - t.Run("suppresses vulnerable provider when registrar context missing", func(t *testing.T) { - evidence := NSEvidence{ - Domain: "a.example.ca", - NSHosts: []string{"ns1.risky-dns.net"}, - NSDelegations: model.NsDelegations{Delegation: model.Delegation{ - LameType: "full", - }}, - } - - got := MatchNSProviderRules(evidence, nsFP) - if got == nil { - t.Fatal("expected hit, got nil") - } - if got.ReasonCode != ReasonNSRegistrarContextInsufficient { - t.Fatalf("unexpected reason: %q", got.ReasonCode) - } - if got.Matched { - t.Fatal("expected matched=false when registrar context is missing") - } - }) -} - -func TestShouldEmitNSHijack(t *testing.T) { - tests := []struct { - name string - hit *NSHit - want bool - }{ - {name: "nil hit", hit: nil, want: false}, - {name: "unmatched hit", hit: &NSHit{Matched: false}, want: false}, - {name: "matched hit", hit: &NSHit{Matched: true}, want: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := ShouldEmitNSHijack(tt.hit); got != tt.want { - t.Fatalf("ShouldEmitNSHijack()=%v want=%v", got, tt.want) - } - }) - } -} diff --git a/scanners/subdomain-takeover/internal/detect/registrar_helpers.go b/scanners/subdomain-takeover/internal/detect/registrar_helpers.go index 435d6728e1..356abb535f 100644 --- a/scanners/subdomain-takeover/internal/detect/registrar_helpers.go +++ b/scanners/subdomain-takeover/internal/detect/registrar_helpers.go @@ -56,74 +56,74 @@ func normalizeOrgName(raw string) string { return strings.Join(out, " ") } -func canonicalProviderKey(raw string) string { - var canonicalProviderAliases = map[string][]string{ - "aws-route53": { - "aws route 53", "route 53", "route53", "amazon route 53", "amazon web services", "aws", - }, - "azure-dns": { - "azure", "azure dns", "microsoft", "microsoft azure", - }, - "cloudflare": { - "cloudflare", "cloudflare inc", - }, - "digitalocean": { - "digital ocean", "digitalocean", - }, - "dnsmadeeasy": { - "dns made easy", "dnsmadeeasy", - }, - "dnsimple": { - "dnsimple", - }, - "domaincom": { - "domain com", "domain.com", "domain", - }, - "dreamhost": { - "dreamhost", - }, - "easydns": { - "easydns", "easy dns", - }, - "gandi": { - "gandi", "gandi sas", "gandi.net", - }, - "google-cloud-dns": { - "google cloud", "google cloud dns", "google domains", "googledomains", "google", - }, - "hurricane-electric": { - "hurricane electric", "he net", "he", - }, - "linode": { - "linode", "akamai linode", "akamai", - }, - "namecom": { - "name.com", "name com", "namecom", - }, - "namecheap": { - "namecheap", - }, - "network-solutions": { - "network solutions", "web.com", "webcom", - }, - "ns1": { - "ns1", "nsone", "nsone.net", - }, - "reg-ru": { - "reg ru", "reg.ru", - }, - "tierranet": { - "tierranet", "tierra net", "domaindiscover", - }, - "ultradns": { - "ultradns", "neustar ultradns", "neustar", - }, - "yahoo-smb": { - "yahoo small business", "yahoo", "yns", - }, - } +var canonicalProviderAliases = map[string][]string{ + "aws-route53": { + "aws route 53", "route 53", "route53", "amazon route 53", "amazon web services", "aws", + }, + "azure-dns": { + "azure", "azure dns", "microsoft", "microsoft azure", + }, + "cloudflare": { + "cloudflare", "cloudflare inc", + }, + "digitalocean": { + "digital ocean", "digitalocean", + }, + "dnsmadeeasy": { + "dns made easy", "dnsmadeeasy", + }, + "dnsimple": { + "dnsimple", + }, + "domaincom": { + "domain com", "domain.com", "domain", + }, + "dreamhost": { + "dreamhost", + }, + "easydns": { + "easydns", "easy dns", + }, + "gandi": { + "gandi", "gandi sas", "gandi.net", + }, + "google-cloud-dns": { + "google cloud", "google cloud dns", "google domains", "googledomains", "google", + }, + "hurricane-electric": { + "hurricane electric", "he net", "he", + }, + "linode": { + "linode", "akamai linode", "akamai", + }, + "namecom": { + "name.com", "name com", "namecom", + }, + "namecheap": { + "namecheap", + }, + "network-solutions": { + "network solutions", "web.com", "webcom", + }, + "ns1": { + "ns1", "nsone", "nsone.net", + }, + "reg-ru": { + "reg ru", "reg.ru", + }, + "tierranet": { + "tierranet", "tierra net", "domaindiscover", + }, + "ultradns": { + "ultradns", "neustar ultradns", "neustar", + }, + "yahoo-smb": { + "yahoo small business", "yahoo", "yns", + }, +} +func canonicalProviderKey(raw string) string { s := normalizeOrgName(raw) // lowercase, trim, collapse spaces, strip punctuation/legal suffixes if s == "" { return "" diff --git a/scanners/subdomain-takeover/internal/messaging/health.go b/scanners/subdomain-takeover/internal/messaging/health.go index 551e72d825..593ca1faff 100644 --- a/scanners/subdomain-takeover/internal/messaging/health.go +++ b/scanners/subdomain-takeover/internal/messaging/health.go @@ -7,21 +7,10 @@ import ( "github.com/nats-io/nats.go" ) -type natsConnectionState interface { - IsClosed() bool - IsConnected() bool - Status() nats.Status -} - func CheckConnection(nc *nats.Conn) error { if nc == nil { return errors.New("nats connection is nil") } - - return checkConnectionState(nc) -} - -func checkConnectionState(nc natsConnectionState) error { if nc.IsClosed() { return errors.New("nats connection is closed") } diff --git a/scanners/subdomain-takeover/internal/messaging/health_test.go b/scanners/subdomain-takeover/internal/messaging/health_test.go deleted file mode 100644 index 06829d6757..0000000000 --- a/scanners/subdomain-takeover/internal/messaging/health_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package messaging - -import ( - "testing" - - "github.com/nats-io/nats.go" -) - -type fakeConnState struct { - closed bool - connected bool - status nats.Status -} - -func (f fakeConnState) IsClosed() bool { return f.closed } -func (f fakeConnState) IsConnected() bool { return f.connected } -func (f fakeConnState) Status() nats.Status { - return f.status -} - -func TestCheckConnection(t *testing.T) { - err := CheckConnection(nil) - if err == nil { - t.Fatal("expected error for nil connection") - } - if err.Error() != "nats connection is nil" { - t.Fatalf("unexpected error: %v", err) - } -} - -func TestCheckConnectionState(t *testing.T) { - t.Run("closed connection returns closed error", func(t *testing.T) { - err := checkConnectionState(fakeConnState{closed: true}) - if err == nil || err.Error() != "nats connection is closed" { - t.Fatalf("unexpected error: %v", err) - } - }) - - t.Run("not connected returns status error", func(t *testing.T) { - err := checkConnectionState(fakeConnState{connected: false, status: nats.DISCONNECTED}) - if err == nil || err.Error() != "nats not connected (status=DISCONNECTED)" { - t.Fatalf("unexpected error: %v", err) - } - }) - - t.Run("connected state returns nil", func(t *testing.T) { - err := checkConnectionState(fakeConnState{connected: true, status: nats.CONNECTED}) - if err != nil { - t.Fatalf("expected nil error, got: %v", err) - } - }) -} diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go index bd43b9c2d4..7364300d5e 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -11,25 +11,18 @@ import ( "github.com/rs/zerolog" ) -var marshalFinding = json.Marshal -var nowUTC = func() time.Time { return time.Now().UTC() } - type Publisher struct { - logger zerolog.Logger - publishToNATS func(ctx context.Context, subj string, data []byte, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error) - subject string + logger zerolog.Logger + js jetstream.JetStream + subject string } -func NewPublisher( - logger zerolog.Logger, - publishToNATS func(ctx context.Context, subj string, data []byte, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error), - subject string, -) *Publisher { - return &Publisher{logger: logger, publishToNATS: publishToNATS, subject: subject} +func NewPublisher(logger zerolog.Logger, js jetstream.JetStream, subject string) *Publisher { + return &Publisher{logger: logger, js: js, subject: subject} } func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { - event, err := model.NewFindingEventFromFinding(finding, nowUTC()) + event, err := model.NewFindingEventFromFinding(finding, time.Now().UTC()) if err != nil { p.logger.Error(). Err(err). @@ -40,7 +33,7 @@ func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { return err } - payload, err := marshalFinding(event) + payload, err := json.Marshal(event) if err != nil { p.logger.Error(). Err(err). @@ -54,7 +47,7 @@ func (p *Publisher) Publish(ctx context.Context, finding model.Finding) error { return err } - if _, err := p.publishToNATS(ctx, p.subject, payload); err != nil { + if _, err := p.js.Publish(ctx, p.subject, payload); err != nil { p.logger.Error(). Err(err). Str("domain", finding.Domain). diff --git a/scanners/subdomain-takeover/internal/messaging/publisher_test.go b/scanners/subdomain-takeover/internal/messaging/publisher_test.go deleted file mode 100644 index 785a372663..0000000000 --- a/scanners/subdomain-takeover/internal/messaging/publisher_test.go +++ /dev/null @@ -1,199 +0,0 @@ -package messaging - -import ( - "context" - "encoding/json" - "errors" - "testing" - "time" - - "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" - "github.com/nats-io/nats.go/jetstream" - "github.com/rs/zerolog" -) - -type fakePublishClient struct { - err error - ack *jetstream.PubAck - called int - lastSubj string - lastBytes []byte -} - -func (f *fakePublishClient) Publish(_ context.Context, subj string, data []byte, _ ...jetstream.PublishOpt) (*jetstream.PubAck, error) { - f.called++ - f.lastSubj = subj - f.lastBytes = append([]byte(nil), data...) - if f.err != nil { - return nil, f.err - } - if f.ack != nil { - return f.ack, nil - } - return &jetstream.PubAck{}, nil -} - -func TestNewPublisher(t *testing.T) { - client := &fakePublishClient{} - p := NewPublisher(zerolog.Nop(), client.Publish, "scans.findings.subdomain-takeover") - if p == nil { - t.Fatal("expected publisher instance") - } - if p.subject != "scans.findings.subdomain-takeover" { - t.Fatalf("unexpected subject: %q", p.subject) - } -} - -func TestPublisherPublish(t *testing.T) { - origMarshal := marshalFinding - origNowUTC := nowUTC - t.Cleanup(func() { marshalFinding = origMarshal }) - t.Cleanup(func() { nowUTC = origNowUTC }) - - fixedNow := time.Date(2026, 8, 7, 12, 34, 56, 0, time.UTC) - nowUTC = func() time.Time { return fixedNow } - - nsFinding := model.Finding{ - Domain: "a.example.ca", - DomainKey: "123", - RecordType: model.RecordTypeNS, - Target: "ns1.risky-dns.net", - Provider: "RiskyDNS", - Confidence: "probable", - ReasonCode: "NS_PARTIAL_LAME_PROVIDER_VULNERABLE", - } - cnameFinding := model.Finding{ - Domain: "b.example.ca", - DomainKey: "456", - RecordType: model.RecordTypeCNAME, - Target: "old-app.azurewebsites.net", - Provider: "azure", - Confidence: "probable", - ReasonCode: "CNAME_DANGLING_NXDOMAIN", - } - - t.Run("publishes expected finding event payload", func(t *testing.T) { - tests := []struct { - name string - finding model.Finding - wantFindingType string - }{ - { - name: "ns record", - finding: nsFinding, - wantFindingType: "subdomain-takeover-ns", - }, - { - name: "cname record", - finding: cnameFinding, - wantFindingType: "subdomain-takeover-cname", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - client := &fakePublishClient{} - p := &Publisher{logger: zerolog.Nop(), publishToNATS: client.Publish, subject: "scans.findings.subdomain-takeover"} - - err := p.Publish(context.Background(), tc.finding) - if err != nil { - t.Fatalf("unexpected publish error: %v", err) - } - if client.called != 1 { - t.Fatalf("expected one publish call, got %d", client.called) - } - if client.lastSubj != "scans.findings.subdomain-takeover" { - t.Fatalf("unexpected subject: %q", client.lastSubj) - } - - var got model.FindingEvent - if err := json.Unmarshal(client.lastBytes, &got); err != nil { - t.Fatalf("payload not valid json: %v", err) - } - - if got.Source != "subdomain-takeover" { - t.Fatalf("unexpected source: %q", got.Source) - } - if got.FindingType != tc.wantFindingType { - t.Fatalf("unexpected findingType: %q", got.FindingType) - } - if got.DomainKey != tc.finding.DomainKey { - t.Fatalf("unexpected domainKey: %q", got.DomainKey) - } - if got.Subject != tc.finding.Domain { - t.Fatalf("unexpected subject: %q", got.Subject) - } - if got.ObservedAt != fixedNow.Format(time.RFC3339) { - t.Fatalf("unexpected observedAt: %q", got.ObservedAt) - } - if got.Evidence["target"] != tc.finding.Target { - t.Fatalf("unexpected evidence target: %v", got.Evidence["target"]) - } - if got.Evidence["recordType"] != string(tc.finding.RecordType) { - t.Fatalf("unexpected evidence recordType: %v", got.Evidence["recordType"]) - } - if got.Attributes["provider"] != tc.finding.Provider { - t.Fatalf("unexpected attributes provider: %v", got.Attributes["provider"]) - } - if got.Attributes["lameType"] != tc.finding.LameType { - t.Fatalf("unexpected attributes lameType: %v", got.Attributes["lameType"]) - } - }) - } - }) - - t.Run("propagates publish client error", func(t *testing.T) { - client := &fakePublishClient{err: errors.New("publish failed")} - p := &Publisher{logger: zerolog.Nop(), publishToNATS: client.Publish, subject: "scans.findings.subdomain-takeover"} - - err := p.Publish(context.Background(), nsFinding) - if err == nil { - t.Fatal("expected publish error") - } - if client.called != 1 { - t.Fatalf("expected one publish call, got %d", client.called) - } - }) - - t.Run("returns mapping or marshal errors without publish", func(t *testing.T) { - tests := []struct { - name string - finding model.Finding - marshalErr error - }{ - { - name: "marshal error", - finding: nsFinding, - marshalErr: errors.New("marshal failed"), - }, - { - name: "unsupported record type mapping error", - finding: func() model.Finding { - badFinding := nsFinding - badFinding.RecordType = model.RecordType("TXT") - return badFinding - }(), - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - marshalFinding = origMarshal - if tc.marshalErr != nil { - marshalFinding = func(any) ([]byte, error) { return nil, tc.marshalErr } - } - - client := &fakePublishClient{} - p := &Publisher{logger: zerolog.Nop(), publishToNATS: client.Publish, subject: "scans.findings.subdomain-takeover"} - - err := p.Publish(context.Background(), tc.finding) - if err == nil { - t.Fatal("expected error") - } - if client.called != 0 { - t.Fatalf("expected no publish call, got %d", client.called) - } - }) - } - }) -} diff --git a/scanners/subdomain-takeover/internal/model/finding.go b/scanners/subdomain-takeover/internal/model/finding.go index 7563a658f9..e89b1f6cbb 100644 --- a/scanners/subdomain-takeover/internal/model/finding.go +++ b/scanners/subdomain-takeover/internal/model/finding.go @@ -1,6 +1,9 @@ package model -import "time" +import ( + "errors" + "time" +) type FindingType string @@ -75,10 +78,4 @@ func FindingTypeForRecord(recordType RecordType) (string, error) { } } -var ErrUnsupportedRecordType = unsupportedRecordTypeError{} - -type unsupportedRecordTypeError struct{} - -func (unsupportedRecordTypeError) Error() string { - return "unsupported record type" -} +var ErrUnsupportedRecordType = errors.New("unsupported record type") From 3e294865078f9d07d23ee4dcfd6207390d2c90e0 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Mon, 31 Aug 2026 11:19:06 -0300 Subject: [PATCH 63/65] test(subdomain-takeover): rewrite unit tests against lean prod APIs --- .../internal/app/runner_test.go | 87 ++++++++ .../internal/app/worker_test.go | 195 ++++++++++++++++++ .../internal/detect/classifier_test.go | 134 ++++++++++++ .../internal/detect/cname_rules_test.go | 116 +++++++++++ .../internal/detect/ns_reasoning_test.go | 137 ++++++++++++ .../internal/detect/ns_rules_test.go | 169 +++++++++++++++ .../internal/messaging/health_test.go | 27 +++ .../internal/messaging/publisher.go | 4 +- .../internal/messaging/publisher_test.go | 143 +++++++++++++ 9 files changed, 1010 insertions(+), 2 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/app/runner_test.go create mode 100644 scanners/subdomain-takeover/internal/app/worker_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/classifier_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/cname_rules_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go create mode 100644 scanners/subdomain-takeover/internal/detect/ns_rules_test.go create mode 100644 scanners/subdomain-takeover/internal/messaging/health_test.go create mode 100644 scanners/subdomain-takeover/internal/messaging/publisher_test.go diff --git a/scanners/subdomain-takeover/internal/app/runner_test.go b/scanners/subdomain-takeover/internal/app/runner_test.go new file mode 100644 index 0000000000..7f74cd791b --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/runner_test.go @@ -0,0 +1,87 @@ +package app + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type fakeMessagesIter struct { + mu sync.Mutex + msgs []jetstream.Msg + err error + idx int +} + +func (f *fakeMessagesIter) Next(...jetstream.NextOpt) (jetstream.Msg, error) { + f.mu.Lock() + defer f.mu.Unlock() + + if f.err != nil { + return nil, f.err + } + if f.idx >= len(f.msgs) { + return nil, jetstream.ErrMsgIteratorClosed + } + m := f.msgs[f.idx] + f.idx++ + return m, nil +} + +func (f *fakeMessagesIter) Stop() {} +func (f *fakeMessagesIter) Drain() {} + +func TestIsBenignNextError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {name: "iterator closed", err: jetstream.ErrMsgIteratorClosed, want: true}, + {name: "no messages", err: jetstream.ErrNoMessages, want: true}, + {name: "nats timeout", err: nats.ErrTimeout, want: true}, + {name: "deadline exceeded", err: context.DeadlineExceeded, want: true}, + {name: "context canceled is not benign", err: context.Canceled, want: false}, + {name: "other error", err: errors.New("boom"), want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isBenignNextError(tt.err); got != tt.want { + t.Fatalf("isBenignNextError(%v)=%v want=%v", tt.err, got, tt.want) + } + }) + } +} + +// Run's connection health check requires a real *nats.Conn to exercise the +// "healthy" path (nats.Conn has no exported way to fake IsConnected()==true), +// so message-processing behavior is covered by TestWorkerHandle instead. What +// is covered here at the unit level is the unhealthy-connection short-circuit. +func TestRun_ReturnsImmediatelyWhenConnectionUnhealthy(t *testing.T) { + iter := &fakeMessagesIter{} + classifier := detect.NewClassifier(nil, zerolog.Nop()) + publisher := messaging.NewPublisher(zerolog.Nop(), &fakeJSPublishClient{}, "scans.findings.subdomain-takeover") + worker := NewWorker(zerolog.Nop(), publisher, classifier) + + deps := RunnerDeps{ + Logger: zerolog.Nop(), + WorkerCount: 2, + Iter: iter, + Worker: worker, + NC: nil, + } + + Run(context.Background(), deps) + + if iter.idx != 0 { + t.Fatalf("expected no iterator reads, got %d", iter.idx) + } +} diff --git a/scanners/subdomain-takeover/internal/app/worker_test.go b/scanners/subdomain-takeover/internal/app/worker_test.go new file mode 100644 index 0000000000..06d87413b9 --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/worker_test.go @@ -0,0 +1,195 @@ +package app + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type fakeJSPublishClient struct { + err error + called int +} + +func (f *fakeJSPublishClient) Publish(context.Context, string, []byte, ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + f.called++ + if f.err != nil { + return nil, f.err + } + return &jetstream.PubAck{}, nil +} +func (f *fakeJSPublishClient) PublishMsg(context.Context, *nats.Msg, ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + return nil, nil +} +func (f *fakeJSPublishClient) PublishAsync(string, []byte, ...jetstream.PublishOpt) (jetstream.PubAckFuture, error) { + return nil, nil +} +func (f *fakeJSPublishClient) PublishMsgAsync(*nats.Msg, ...jetstream.PublishOpt) (jetstream.PubAckFuture, error) { + return nil, nil +} +func (f *fakeJSPublishClient) PublishAsyncPending() int { return 0 } +func (f *fakeJSPublishClient) PublishAsyncComplete() <-chan struct{} { return nil } +func (f *fakeJSPublishClient) CleanupPublisher() {} + +type fakeJSMsg struct { + data []byte + subject string + + ackErr error + nakErr error + termErr error + + ackCount int + nakCount int + termCount int +} + +func (m *fakeJSMsg) Metadata() (*jetstream.MsgMetadata, error) { return nil, nil } +func (m *fakeJSMsg) Data() []byte { return m.data } +func (m *fakeJSMsg) Headers() nats.Header { return nil } +func (m *fakeJSMsg) Subject() string { return m.subject } +func (m *fakeJSMsg) Reply() string { return "" } + +func (m *fakeJSMsg) Ack() error { + m.ackCount++ + return m.ackErr +} + +func (m *fakeJSMsg) DoubleAck(context.Context) error { return nil } + +func (m *fakeJSMsg) Nak() error { + m.nakCount++ + return m.nakErr +} + +func (m *fakeJSMsg) NakWithDelay(time.Duration) error { return nil } +func (m *fakeJSMsg) InProgress() error { return nil } + +func (m *fakeJSMsg) Term() error { + m.termCount++ + return m.termErr +} + +func (m *fakeJSMsg) TermWithReason(string) error { return nil } + +// scanWithFinding is DNS scan input that the real classifier turns into exactly +// one CNAME finding (a dangling Azure Web Apps CNAME), so Worker.Handle's +// publish path can be exercised without inventing a classifier seam. +const scanWithFinding = `{"domain_key":"k","results":{"domain":"a.example.ca","cname_record":"a.example.ca. 300 IN CNAME foo.azurewebsites.net."}}` + +const scanWithNoFinding = `{"domain_key":"k","results":{}}` + +func TestWorkerHandle(t *testing.T) { + if err := fingerprints.Load(zerolog.Nop()); err != nil { + t.Fatalf("failed to load fingerprints: %v", err) + } + logger := zerolog.Nop() + classifier := detect.NewClassifier(nil, logger) + + tests := []struct { + name string + data []byte + ackErr error + nakErr error + termErr error + publisherErr error + wantErr bool + wantAck int + wantNak int + wantTerm int + wantPublishCall int + }{ + { + name: "decode error terminates message", + data: []byte("{not-json"), + wantErr: true, + wantTerm: 1, + }, + { + name: "publish error naks message", + data: []byte(scanWithFinding), + publisherErr: errors.New("publish failed"), + wantErr: true, + wantNak: 1, + wantPublishCall: 1, + }, + { + name: "successful processing publishes finding and acks", + data: []byte(scanWithFinding), + wantErr: false, + wantAck: 1, + wantPublishCall: 1, + }, + { + name: "no findings still acks without publishing", + data: []byte(scanWithNoFinding), + wantErr: false, + wantAck: 1, + }, + { + name: "ack failure returns error", + data: []byte(scanWithNoFinding), + ackErr: errors.New("ack failed"), + wantErr: true, + wantAck: 1, + }, + { + name: "decode error still returned when term fails", + data: []byte("{bad-json"), + termErr: errors.New("term failed"), + wantErr: true, + wantTerm: 1, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + client := &fakeJSPublishClient{err: tc.publisherErr} + publisher := messaging.NewPublisher(logger, client, "scans.findings.subdomain-takeover") + worker := NewWorker(logger, publisher, classifier) + + msg := &fakeJSMsg{ + data: tc.data, + subject: "scans.dns_scanner_results", + ackErr: tc.ackErr, + nakErr: tc.nakErr, + termErr: tc.termErr, + } + + err := worker.Handle(context.Background(), msg) + if tc.wantErr && err == nil { + t.Fatal("expected error") + } + if !tc.wantErr && err != nil { + t.Fatalf("unexpected error: %v", err) + } + if msg.ackCount != tc.wantAck { + t.Fatalf("unexpected ack count: got=%d want=%d", msg.ackCount, tc.wantAck) + } + if msg.nakCount != tc.wantNak { + t.Fatalf("unexpected nak count: got=%d want=%d", msg.nakCount, tc.wantNak) + } + if msg.termCount != tc.wantTerm { + t.Fatalf("unexpected term count: got=%d want=%d", msg.termCount, tc.wantTerm) + } + if client.called != tc.wantPublishCall { + t.Fatalf("unexpected publish call count: got=%d want=%d", client.called, tc.wantPublishCall) + } + }) + } +} + +func TestDecodeScan_TrimsTrailingNewline(t *testing.T) { + _, err := decodeScan([]byte("{\"domain_key\":\"k\",\"results\":{}}\n")) + if err != nil { + t.Fatalf("expected decode success, got error: %v", err) + } +} diff --git a/scanners/subdomain-takeover/internal/detect/classifier_test.go b/scanners/subdomain-takeover/internal/detect/classifier_test.go new file mode 100644 index 0000000000..8cd07a538a --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/classifier_test.go @@ -0,0 +1,134 @@ +package detect + +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/rs/zerolog" +) + +func TestClassifierClassify(t *testing.T) { + if err := fingerprints.Load(zerolog.Nop()); err != nil { + t.Fatalf("failed to load fingerprints: %v", err) + } + + t.Run("emits cname and ns findings when both are exploitable", func(t *testing.T) { + rdapMatch := true + input := model.Input{ + DomainKey: "k1", + Results: model.ScanResults{ + Domain: strPtr("a.example.ca"), + CnameRecord: strPtr("a.example.ca. 300 IN CNAME foo.azurewebsites.net."), + RegistrarContext: &model.RegistrarContext{ + LookupSuccess: true, + RegistrarName: "Namecheap", + DelegationMatchesRDAP: &rdapMatch, + }, + NsDelegations: &model.NsDelegations{ + Hosts: []string{"ns1.digitalocean.com"}, + Delegation: model.Delegation{ + LameType: "partial", + }, + }, + }, + } + + classifier := NewClassifier(fakeMatcher{}, zerolog.Nop()) + findings, err := classifier.Classify(input) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + if len(findings) != 2 { + t.Fatalf("expected 2 findings, got %d", len(findings)) + } + + var sawCNAME, sawNS bool + for _, f := range findings { + if f.RecordType == model.RecordTypeCNAME { + sawCNAME = true + if f.ReasonCode != string(ReasonCNAMEDanglingNXDOMAIN) { + t.Fatalf("unexpected cname reason: %q", f.ReasonCode) + } + if f.Confidence != ConfidenceProbable { + t.Fatalf("unexpected cname confidence: %q", f.Confidence) + } + } + if f.RecordType == model.RecordTypeNS { + sawNS = true + if f.ReasonCode != string(ReasonNSPartialLameProviderVulnerable) { + t.Fatalf("unexpected ns reason: %q", f.ReasonCode) + } + if f.Confidence != ConfidenceProbable { + t.Fatalf("unexpected ns confidence: %q", f.Confidence) + } + } + } + if !sawCNAME || !sawNS { + t.Fatalf("missing expected finding types: cname=%v ns=%v", sawCNAME, sawNS) + } + }) + + t.Run("suppresses non-exploitable ns matches", func(t *testing.T) { + input := model.Input{ + DomainKey: "k2", + Results: model.ScanResults{ + Domain: strPtr("b.example.ca"), + CnameRecord: strPtr("b.example.ca. 300 IN CNAME foo.ghost.io."), + NsDelegations: &model.NsDelegations{ + Hosts: []string{"aria.ns.cloudflare.com"}, + Delegation: model.Delegation{ + LameType: "full", + }, + }, + }, + } + + matcher := fakeMatcher{containsFn: func(string, string, fingerprints.FingerprintMode) bool { return true }} + classifier := NewClassifier(matcher, zerolog.Nop()) + findings, err := classifier.Classify(input) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + + if len(findings) != 1 { + t.Fatalf("expected 1 finding, got %d", len(findings)) + } + if findings[0].RecordType != model.RecordTypeCNAME { + t.Fatalf("expected cname-only finding, got %s", findings[0].RecordType) + } + if findings[0].ReasonCode != string(ReasonCNAMEProviderFingerprintBodyMatch) { + t.Fatalf("unexpected cname reason: %q", findings[0].ReasonCode) + } + }) + + t.Run("returns no findings when evidence absent", func(t *testing.T) { + input := model.Input{DomainKey: "k3", Results: model.ScanResults{}} + classifier := NewClassifier(fakeMatcher{}, zerolog.Nop()) + findings, err := classifier.Classify(input) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + if len(findings) != 0 { + t.Fatalf("expected no findings, got %d", len(findings)) + } + }) + + t.Run("defaults to a no-op matcher when none is given", func(t *testing.T) { + classifier := NewClassifier(nil, zerolog.Nop()) + input := model.Input{ + DomainKey: "k4", + Results: model.ScanResults{ + Domain: strPtr("c.example.ca"), + CnameRecord: strPtr("c.example.ca. 300 IN CNAME foo.ghost.io."), + }, + } + findings, err := classifier.Classify(input) + if err != nil { + t.Fatalf("Classify error: %v", err) + } + if len(findings) != 0 { + t.Fatalf("expected body-fingerprint match to be suppressed without a matcher, got %d findings", len(findings)) + } + }) +} diff --git a/scanners/subdomain-takeover/internal/detect/cname_rules_test.go b/scanners/subdomain-takeover/internal/detect/cname_rules_test.go new file mode 100644 index 0000000000..025377bfd6 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/cname_rules_test.go @@ -0,0 +1,116 @@ +package detect + +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/rs/zerolog" +) + +func TestMatchCNAMEFingerprints(t *testing.T) { + fps := []fingerprints.CNAMEProviderFingerprint{ + { + Name: "Azure", + Cname: []string{"azurewebsites.net"}, + Nxdomain: true, + Fingerprint: "This web app is stopped", + }, + { + Name: "Ghost", + Cname: []string{"ghost.io"}, + Nxdomain: false, + Fingerprint: "The thing you were looking for is no longer here", + Mode: fingerprints.FingerprintModeLiteral, + }, + } + + t.Run("returns nil when no target match", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "foo.not-a-provider.net", NoResolve: true} + if got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{}, zerolog.Nop()); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("nxdomain provider emits dangling reason when unresolved", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "foo.azurewebsites.net", NoResolve: true} + got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{}, zerolog.Nop()) + if got == nil { + t.Fatal("expected hit, got nil") + } + if !got.Matched { + t.Fatal("expected matched=true") + } + if got.ReasonCode != ReasonCNAMEDanglingNXDOMAIN { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("nxdomain provider suppressed when resolve evidence exists", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "foo.azurewebsites.net", NoResolve: false} + got := MatchCNAMEFingerprints(evidence, fps, fakeMatcher{}, zerolog.Nop()) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Matched { + t.Fatal("expected matched=false") + } + if got.ReasonCode != ReasonCNAMETargetMatchMissingNXDOMAIN { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("body fingerprint provider emits when matcher returns true", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "blog.ghost.io", NoResolve: false} + matcher := fakeMatcher{containsFn: func(domain string, fingerprint string, mode fingerprints.FingerprintMode) bool { + if domain != "a.example.ca" || mode != fingerprints.FingerprintModeLiteral { + t.Fatalf("unexpected matcher args: domain=%q mode=%q", domain, mode) + } + return true + }} + got := MatchCNAMEFingerprints(evidence, fps, matcher, zerolog.Nop()) + if got == nil { + t.Fatal("expected hit, got nil") + } + if !got.Matched { + t.Fatal("expected matched=true") + } + if got.ReasonCode != ReasonCNAMEProviderFingerprintBodyMatch { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("body fingerprint provider suppressed when matcher false", func(t *testing.T) { + evidence := CNAMEEvidence{Domain: "a.example.ca", Target: "blog.ghost.io", NoResolve: false} + matcher := fakeMatcher{containsFn: func(string, string, fingerprints.FingerprintMode) bool { return false }} + got := MatchCNAMEFingerprints(evidence, fps, matcher, zerolog.Nop()) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Matched { + t.Fatal("expected matched=false") + } + if got.ReasonCode != ReasonCNAMETargetMatchMissingBodyFP { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) +} + +func TestShouldEmitCNAME(t *testing.T) { + tests := []struct { + name string + hit *CNAMEHit + want bool + }{ + {name: "nil hit", hit: nil, want: false}, + {name: "unmatched hit", hit: &CNAMEHit{Matched: false}, want: false}, + {name: "matched hit", hit: &CNAMEHit{Matched: true}, want: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ShouldEmitCNAME(tt.hit); got != tt.want { + t.Fatalf("ShouldEmitCNAME()=%v want=%v", got, tt.want) + } + }) + } +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go new file mode 100644 index 0000000000..4483d2459d --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_reasoning_test.go @@ -0,0 +1,137 @@ +package detect + +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" +) + +func TestGetNSHijackReasonCode(t *testing.T) { + namecheapRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"} + digitalOceanRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."} + invalidRegistrar := &model.RegistrarContext{LookupSuccess: false, RegistrarName: "Namecheap"} + + tests := []struct { + name string + lameType string + provider fingerprints.NSProviderFingerprint + registrar *model.RegistrarContext + want ReasonCode + }{ + { + name: "full vulnerable with registrar mismatch", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSFullLameProviderVulnerable, + }, + { + name: "partial vulnerable with registrar mismatch", + lameType: "partial", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSPartialLameProviderVulnerable, + }, + { + name: "full vulnerable with purchase and mismatch", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerableWithPurchase}, + registrar: namecheapRegistrar, + want: ReasonNSFullLameProviderVulnerable, + }, + { + name: "full non vulnerable", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusNotVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSLameProviderUnknown, + }, + { + name: "unknown lame type", + lameType: "none", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: namecheapRegistrar, + want: ReasonNSProviderMatchOnly, + }, + { + name: "same registrar/provider suppressed", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: digitalOceanRegistrar, + want: ReasonNSRegistrarProviderMatch, + }, + { + name: "missing registrar context suppressed", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: nil, + want: ReasonNSRegistrarContextInsufficient, + }, + { + name: "invalid registrar context suppressed", + lameType: "full", + provider: fingerprints.NSProviderFingerprint{Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable}, + registrar: invalidRegistrar, + want: ReasonNSRegistrarContextInsufficient, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getNSHijackReasonCode(tt.lameType, tt.provider, tt.registrar) + if got != tt.want { + t.Fatalf("getNSHijackReasonCode()=%q want=%q", got, tt.want) + } + }) + } +} + +func TestNSReasoningHelpers(t *testing.T) { + t.Run("isExploitableProviderStatus", func(t *testing.T) { + tests := []struct { + name string + status fingerprints.NSProviderStatus + want bool + }{ + {name: "vulnerable", status: fingerprints.NSStatusVulnerable, want: true}, + {name: "vulnerable with purchase", status: fingerprints.NSStatusVulnerableWithPurchase, want: true}, + {name: "not vulnerable", status: fingerprints.NSStatusNotVulnerable, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isExploitableProviderStatus(tt.status); got != tt.want { + t.Fatalf("isExploitableProviderStatus(%q)=%v want=%v", tt.status, got, tt.want) + } + }) + } + }) + + t.Run("normalizeLameType", func(t *testing.T) { + if got := normalizeLameType(" Partial "); got != "partial" { + t.Fatalf("normalizeLameType mismatch: %q", got) + } + }) + + t.Run("nsReasonRank ordering", func(t *testing.T) { + checks := []struct { + name string + high ReasonCode + low ReasonCode + label string + }{ + {name: "full above partial", high: ReasonNSFullLameProviderVulnerable, low: ReasonNSPartialLameProviderVulnerable, label: "full lame > partial lame"}, + {name: "partial above registrar-provider-match", high: ReasonNSPartialLameProviderVulnerable, low: ReasonNSRegistrarProviderMatch, label: "partial lame > registrar-provider-match"}, + {name: "registrar gate above provider only", high: ReasonNSRegistrarProviderMatch, low: ReasonNSProviderMatchOnly, label: "registrar gate > provider-only"}, + } + + for _, check := range checks { + t.Run(check.name, func(t *testing.T) { + if nsReasonRank(check.high) <= nsReasonRank(check.low) { + t.Fatalf("expected %s", check.label) + } + }) + } + }) +} diff --git a/scanners/subdomain-takeover/internal/detect/ns_rules_test.go b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go new file mode 100644 index 0000000000..59880f2ab5 --- /dev/null +++ b/scanners/subdomain-takeover/internal/detect/ns_rules_test.go @@ -0,0 +1,169 @@ +package detect + +import ( + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/rs/zerolog" +) + +func TestMatchNSProviderRules(t *testing.T) { + nsFP := []fingerprints.NSProviderFingerprint{ + {Name: "UnknownDNS", Status: fingerprints.NSStatusNotVulnerable, HostPatterns: []string{"*.unknown-dns.net"}}, + {Name: "Digital Ocean", Status: fingerprints.NSStatusVulnerable, HostPatterns: []string{"*.risky-dns.net"}}, + } + + namecheapRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "Namecheap"} + digitalOceanRegistrar := &model.RegistrarContext{LookupSuccess: true, RegistrarName: "DigitalOcean, Inc."} + + t.Run("returns nil for missing hosts", func(t *testing.T) { + evidence := NSEvidence{Domain: "a.example.ca"} + if got := MatchNSProviderRules(evidence, nsFP, zerolog.Nop()); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil for missing fingerprints", func(t *testing.T) { + evidence := NSEvidence{Domain: "a.example.ca", NSHosts: []string{"ns1.risky-dns.net"}} + if got := MatchNSProviderRules(evidence, nil, zerolog.Nop()); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("returns nil when no provider matches", func(t *testing.T) { + evidence := NSEvidence{Domain: "a.example.ca", NSHosts: []string{"ns1.nomatch.net"}} + if got := MatchNSProviderRules(evidence, nsFP, zerolog.Nop()); got != nil { + t.Fatalf("expected nil, got %+v", got) + } + }) + + t.Run("matches vulnerable full lame and emits correct reason", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.risky-dns.net"}, + Registrar: namecheapRegistrar, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + got := MatchNSProviderRules(evidence, nsFP, zerolog.Nop()) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Provider != "Digital Ocean" { + t.Fatalf("unexpected provider: %q", got.Provider) + } + if got.ReasonCode != ReasonNSFullLameProviderVulnerable { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if !got.Matched { + t.Fatal("expected matched=true") + } + }) + + t.Run("ranks vulnerable higher than unknown", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.unknown-dns.net", "ns1.risky-dns.net"}, + Registrar: namecheapRegistrar, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "partial", + }}, + } + got := MatchNSProviderRules(evidence, nsFP, zerolog.Nop()) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.Provider != "Digital Ocean" { + t.Fatalf("expected risky provider to win ranking, got %q", got.Provider) + } + if got.ReasonCode != ReasonNSPartialLameProviderVulnerable { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + }) + + t.Run("non-vulnerable full lame is classified but not emittable", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.unknown-dns.net"}, + Registrar: namecheapRegistrar, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + got := MatchNSProviderRules(evidence, nsFP, zerolog.Nop()) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.ReasonCode != ReasonNSLameProviderUnknown { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if got.Matched { + t.Fatal("expected matched=false for non-vulnerable state") + } + }) + + t.Run("suppresses vulnerable provider when registrar matches provider", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.risky-dns.net"}, + Registrar: digitalOceanRegistrar, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + + got := MatchNSProviderRules(evidence, nsFP, zerolog.Nop()) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.ReasonCode != ReasonNSRegistrarProviderMatch { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if got.Matched { + t.Fatal("expected matched=false when registrar and provider match") + } + }) + + t.Run("suppresses vulnerable provider when registrar context missing", func(t *testing.T) { + evidence := NSEvidence{ + Domain: "a.example.ca", + NSHosts: []string{"ns1.risky-dns.net"}, + NSDelegations: model.NsDelegations{Delegation: model.Delegation{ + LameType: "full", + }}, + } + + got := MatchNSProviderRules(evidence, nsFP, zerolog.Nop()) + if got == nil { + t.Fatal("expected hit, got nil") + } + if got.ReasonCode != ReasonNSRegistrarContextInsufficient { + t.Fatalf("unexpected reason: %q", got.ReasonCode) + } + if got.Matched { + t.Fatal("expected matched=false when registrar context is missing") + } + }) +} + +func TestShouldEmitNSHijack(t *testing.T) { + tests := []struct { + name string + hit *NSHit + want bool + }{ + {name: "nil hit", hit: nil, want: false}, + {name: "unmatched hit", hit: &NSHit{Matched: false}, want: false}, + {name: "matched hit", hit: &NSHit{Matched: true}, want: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ShouldEmitNSHijack(tt.hit); got != tt.want { + t.Fatalf("ShouldEmitNSHijack()=%v want=%v", got, tt.want) + } + }) + } +} diff --git a/scanners/subdomain-takeover/internal/messaging/health_test.go b/scanners/subdomain-takeover/internal/messaging/health_test.go new file mode 100644 index 0000000000..6d13a02bc4 --- /dev/null +++ b/scanners/subdomain-takeover/internal/messaging/health_test.go @@ -0,0 +1,27 @@ +package messaging + +import ( + "testing" + + "github.com/nats-io/nats.go" +) + +func TestCheckConnection(t *testing.T) { + t.Run("nil connection", func(t *testing.T) { + err := CheckConnection(nil) + if err == nil || err.Error() != "nats connection is nil" { + t.Fatalf("unexpected error: %v", err) + } + }) + + // The "closed" branch (IsClosed()==true) requires a real connect/close + // cycle against a NATS server -- nats.Conn has no exported way to reach + // that state otherwise. Covered by an integration test, not here. + + t.Run("not yet connected", func(t *testing.T) { + err := CheckConnection(&nats.Conn{}) + if err == nil || err.Error() != "nats not connected (status=DISCONNECTED)" { + t.Fatalf("unexpected error: %v", err) + } + }) +} diff --git a/scanners/subdomain-takeover/internal/messaging/publisher.go b/scanners/subdomain-takeover/internal/messaging/publisher.go index 7364300d5e..b85bf058be 100644 --- a/scanners/subdomain-takeover/internal/messaging/publisher.go +++ b/scanners/subdomain-takeover/internal/messaging/publisher.go @@ -13,11 +13,11 @@ import ( type Publisher struct { logger zerolog.Logger - js jetstream.JetStream + js jetstream.Publisher subject string } -func NewPublisher(logger zerolog.Logger, js jetstream.JetStream, subject string) *Publisher { +func NewPublisher(logger zerolog.Logger, js jetstream.Publisher, subject string) *Publisher { return &Publisher{logger: logger, js: js, subject: subject} } diff --git a/scanners/subdomain-takeover/internal/messaging/publisher_test.go b/scanners/subdomain-takeover/internal/messaging/publisher_test.go new file mode 100644 index 0000000000..a7d9a66ae6 --- /dev/null +++ b/scanners/subdomain-takeover/internal/messaging/publisher_test.go @@ -0,0 +1,143 @@ +package messaging + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +type fakePublishClient struct { + err error + called int + lastSubj string + lastBytes []byte +} + +func (f *fakePublishClient) Publish(_ context.Context, subj string, data []byte, _ ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + f.called++ + f.lastSubj = subj + f.lastBytes = append([]byte(nil), data...) + if f.err != nil { + return nil, f.err + } + return &jetstream.PubAck{}, nil +} + +func (f *fakePublishClient) PublishMsg(context.Context, *nats.Msg, ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + return nil, nil +} +func (f *fakePublishClient) PublishAsync(string, []byte, ...jetstream.PublishOpt) (jetstream.PubAckFuture, error) { + return nil, nil +} +func (f *fakePublishClient) PublishMsgAsync(*nats.Msg, ...jetstream.PublishOpt) (jetstream.PubAckFuture, error) { + return nil, nil +} +func (f *fakePublishClient) PublishAsyncPending() int { return 0 } +func (f *fakePublishClient) PublishAsyncComplete() <-chan struct{} { return nil } +func (f *fakePublishClient) CleanupPublisher() {} + +func TestPublisherPublish(t *testing.T) { + nsFinding := model.Finding{ + Domain: "a.example.ca", + DomainKey: "123", + RecordType: model.RecordTypeNS, + Target: "ns1.risky-dns.net", + Provider: "RiskyDNS", + Confidence: "probable", + ReasonCode: "NS_PARTIAL_LAME_PROVIDER_VULNERABLE", + } + cnameFinding := model.Finding{ + Domain: "b.example.ca", + DomainKey: "456", + RecordType: model.RecordTypeCNAME, + Target: "old-app.azurewebsites.net", + Provider: "azure", + Confidence: "probable", + ReasonCode: "CNAME_DANGLING_NXDOMAIN", + } + + t.Run("publishes expected finding event payload", func(t *testing.T) { + tests := []struct { + name string + finding model.Finding + wantFindingType string + }{ + {name: "ns record", finding: nsFinding, wantFindingType: "subdomain-takeover-ns"}, + {name: "cname record", finding: cnameFinding, wantFindingType: "subdomain-takeover-cname"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + client := &fakePublishClient{} + p := NewPublisher(zerolog.Nop(), client, "scans.findings.subdomain-takeover") + + if err := p.Publish(context.Background(), tc.finding); err != nil { + t.Fatalf("unexpected publish error: %v", err) + } + if client.called != 1 { + t.Fatalf("expected one publish call, got %d", client.called) + } + if client.lastSubj != "scans.findings.subdomain-takeover" { + t.Fatalf("unexpected subject: %q", client.lastSubj) + } + + var got model.FindingEvent + if err := json.Unmarshal(client.lastBytes, &got); err != nil { + t.Fatalf("payload not valid json: %v", err) + } + if got.Source != "subdomain-takeover" { + t.Fatalf("unexpected source: %q", got.Source) + } + if got.FindingType != tc.wantFindingType { + t.Fatalf("unexpected findingType: %q", got.FindingType) + } + if got.DomainKey != tc.finding.DomainKey { + t.Fatalf("unexpected domainKey: %q", got.DomainKey) + } + if got.Subject != tc.finding.Domain { + t.Fatalf("unexpected subject: %q", got.Subject) + } + if got.Evidence["target"] != tc.finding.Target { + t.Fatalf("unexpected evidence target: %v", got.Evidence["target"]) + } + if got.Attributes["provider"] != tc.finding.Provider { + t.Fatalf("unexpected attributes provider: %v", got.Attributes["provider"]) + } + }) + } + }) + + t.Run("propagates publish client error", func(t *testing.T) { + client := &fakePublishClient{err: errors.New("publish failed")} + p := NewPublisher(zerolog.Nop(), client, "scans.findings.subdomain-takeover") + + if err := p.Publish(context.Background(), nsFinding); err == nil { + t.Fatal("expected publish error") + } + if client.called != 1 { + t.Fatalf("expected one publish call, got %d", client.called) + } + }) + + t.Run("returns mapping error for unsupported record type without publishing", func(t *testing.T) { + badFinding := nsFinding + badFinding.RecordType = model.RecordType("TXT") + + client := &fakePublishClient{} + p := NewPublisher(zerolog.Nop(), client, "scans.findings.subdomain-takeover") + + err := p.Publish(context.Background(), badFinding) + if !errors.Is(err, model.ErrUnsupportedRecordType) { + t.Fatalf("expected ErrUnsupportedRecordType, got %v", err) + } + if client.called != 0 { + t.Fatalf("expected no publish call, got %d", client.called) + } + }) +} From ac7284a482f4693494cec42c9a47e1256f8ea953 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Mon, 31 Aug 2026 11:43:22 -0300 Subject: [PATCH 64/65] test(subdomain-takeover): add integration tests for NATS server --- scanners/subdomain-takeover/Makefile | 26 +-- scanners/subdomain-takeover/go.mod | 14 +- scanners/subdomain-takeover/go.sum | 21 +++ .../internal/app/run_integration_test.go | 174 ++++++++++++++++++ 4 files changed, 220 insertions(+), 15 deletions(-) create mode 100644 scanners/subdomain-takeover/internal/app/run_integration_test.go diff --git a/scanners/subdomain-takeover/Makefile b/scanners/subdomain-takeover/Makefile index 72345ab0b2..2206e2fd68 100644 --- a/scanners/subdomain-takeover/Makefile +++ b/scanners/subdomain-takeover/Makefile @@ -1,4 +1,4 @@ -.PHONY: help run test test-race build fmt fmt-check vet lint tidy ci +.PHONY: help run test test-race test-integration build fmt fmt-check vet lint tidy ci GO ?= go SERVICE_BIN ?= subdomain-takeover @@ -6,16 +6,17 @@ BUILD_DIR ?= bin help: @printf "Targets:\n" - @printf " make run - Run the service\n" - @printf " make test - Run all tests\n" - @printf " make test-race - Run tests with race detector\n" - @printf " make build - Build service binary\n" - @printf " make fmt - Format Go files\n" - @printf " make fmt-check - Check formatting (no changes)\n" - @printf " make vet - Run go vet\n" - @printf " make lint - Run fmt-check + vet\n" - @printf " make tidy - Tidy modules\n" - @printf " make ci - Lint, test, build\n" + @printf " make run - Run the service\n" + @printf " make test - Run all unit tests\n" + @printf " make test-race - Run unit tests with race detector\n" + @printf " make test-integration - Run integration tests (boots an in-process NATS server)\n" + @printf " make build - Build service binary\n" + @printf " make fmt - Format Go files\n" + @printf " make fmt-check - Check formatting (no changes)\n" + @printf " make vet - Run go vet\n" + @printf " make lint - Run fmt-check + vet\n" + @printf " make tidy - Tidy modules\n" + @printf " make ci - Lint, test, build\n" run: $(GO) run ./cmd/service @@ -26,6 +27,9 @@ test: test-race: $(GO) test -race ./... +test-integration: + $(GO) test -tags integration ./... + build: mkdir -p $(BUILD_DIR) CGO_ENABLED=0 $(GO) build -o $(BUILD_DIR)/$(SERVICE_BIN) ./cmd/service diff --git a/scanners/subdomain-takeover/go.mod b/scanners/subdomain-takeover/go.mod index ca7b20f4ac..5966eedfc8 100644 --- a/scanners/subdomain-takeover/go.mod +++ b/scanners/subdomain-takeover/go.mod @@ -9,11 +9,17 @@ require ( ) require ( - github.com/klauspost/compress v1.18.5 // indirect + github.com/antithesishq/antithesis-sdk-go v0.7.2-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/nats-io/nkeys v0.4.15 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.2 // indirect + github.com/nats-io/nats-server/v2 v2.14.5 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect - golang.org/x/crypto v0.49.0 // indirect - golang.org/x/sys v0.42.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/time v0.15.0 // indirect ) diff --git a/scanners/subdomain-takeover/go.sum b/scanners/subdomain-takeover/go.sum index 470146821e..47dc14f1fc 100644 --- a/scanners/subdomain-takeover/go.sum +++ b/scanners/subdomain-takeover/go.sum @@ -1,21 +1,42 @@ +github.com/antithesishq/antithesis-sdk-go v0.7.2-default-no-op h1:p2zFsAzvhIpFya8AIOHIbWf7NGvO34QpLGclyf7nXj8= +github.com/antithesishq/antithesis-sdk-go v0.7.2-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc= +github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc= +github.com/nats-io/nats-server/v2 v2.14.5 h1:M6yeo/Xb7khi97RSEVELof3DForDqmYza3P4tHCPFWw= +github.com/nats-io/nats-server/v2 v2.14.5/go.mod h1:1D3iocrisKvWaD1B/imqarTqmaGrWMqALMLbEDo3v7Q= github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/scanners/subdomain-takeover/internal/app/run_integration_test.go b/scanners/subdomain-takeover/internal/app/run_integration_test.go new file mode 100644 index 0000000000..6269375192 --- /dev/null +++ b/scanners/subdomain-takeover/internal/app/run_integration_test.go @@ -0,0 +1,174 @@ +//go:build integration + +package app_test + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/app" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/bootstrap" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/config" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/detect" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/fingerprints" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/messaging" + "github.com/canada-ca/tracker/scanners/subdomain-takeover/internal/model" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +// startTestNATS boots a real, in-process NATS server with JetStream enabled +// so Run's connection-health and message-iteration paths can be exercised +// against the genuine client, not a fake -- nats.Conn exposes no way to fake +// a healthy connected state, so this is the only way to reach that code. +func startTestNATS(t *testing.T) string { + t.Helper() + + opts := &server.Options{ + Host: "127.0.0.1", + Port: -1, + JetStream: true, + StoreDir: t.TempDir(), + NoLog: true, + NoSigs: true, + } + + ns, err := server.NewServer(opts) + if err != nil { + t.Fatalf("failed to create test nats server: %v", err) + } + + ns.Start() + t.Cleanup(ns.Shutdown) + + if !ns.ReadyForConnections(5 * time.Second) { + t.Fatal("test nats server did not become ready") + } + + return ns.ClientURL() +} + +func TestIntegration_Run_ProcessesMessageEndToEnd(t *testing.T) { + if err := fingerprints.Load(zerolog.Nop()); err != nil { + t.Fatalf("failed to load fingerprints: %v", err) + } + + url := startTestNATS(t) + logger := zerolog.Nop() + + cfg := &config.Config{ + NATSURL: url, + NATSStream: "SUBDOMAIN_TAKEOVER_TEST", + SubjectIn: "scans.dns_scanner_results.test", + SubjectOut: "scans.findings.subdomain-takeover.test", + DurableName: "subdomain_takeover_test", + WorkerCount: 2, + } + + setupCtx, cancelSetup := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelSetup() + + setupConn, err := nats.Connect(cfg.NATSURL) + if err != nil { + t.Fatalf("failed to connect for setup: %v", err) + } + defer setupConn.Close() + + js, err := jetstream.New(setupConn) + if err != nil { + t.Fatalf("failed to create jetstream client: %v", err) + } + if _, err := js.CreateStream(setupCtx, jetstream.StreamConfig{ + Name: cfg.NATSStream, + Subjects: []string{cfg.SubjectIn, cfg.SubjectOut}, + }); err != nil { + t.Fatalf("failed to create stream: %v", err) + } + + // bootstrap.NewRuntimeDeps is the same real wiring main.go uses -- reusing + // it here means the test exercises the actual consumer/iterator setup, + // not a reimplementation of it. + runtimeDeps, err := bootstrap.NewRuntimeDeps(setupCtx, cfg, logger) + if err != nil { + t.Fatalf("failed to build runtime deps: %v", err) + } + defer runtimeDeps.NC.Close() + + // Subscribing on the core subject observes findings the worker publishes, + // independent of the JetStream storage the assertions below also check. + findings := make(chan *nats.Msg, 1) + sub, err := setupConn.Subscribe(cfg.SubjectOut, func(msg *nats.Msg) { + findings <- msg + }) + if err != nil { + t.Fatalf("failed to subscribe to output subject: %v", err) + } + defer sub.Unsubscribe() + + publisher := messaging.NewPublisher(logger, runtimeDeps.JS, cfg.SubjectOut) + classifier := detect.NewClassifier(nil, logger) + worker := app.NewWorker(logger, publisher, classifier) + + // This input matches the real "Microsoft Azure" CNAME fingerprint's + // NXDOMAIN mode, so the real classifier deterministically emits one + // finding without needing an HTTP body fetch. + scan := model.Input{ + DomainKey: "integration-test", + Results: model.ScanResults{ + Domain: strPtrForTest("dangling.example.ca"), + CnameRecord: strPtrForTest("dangling.example.ca. 300 IN CNAME foo.azurewebsites.net."), + }, + } + payload, err := json.Marshal(scan) + if err != nil { + t.Fatalf("failed to marshal scan input: %v", err) + } + if _, err := js.Publish(setupCtx, cfg.SubjectIn, payload); err != nil { + t.Fatalf("failed to publish scan input: %v", err) + } + + runCtx, cancelRun := context.WithCancel(context.Background()) + runDone := make(chan struct{}) + go func() { + defer close(runDone) + app.Run(runCtx, app.RunnerDeps{ + Logger: logger, + WorkerCount: cfg.WorkerCount, + Iter: runtimeDeps.Iter, + Worker: worker, + NC: runtimeDeps.NC, + }) + }() + + select { + case msg := <-findings: + var event model.FindingEvent + if err := json.Unmarshal(msg.Data, &event); err != nil { + t.Fatalf("finding event payload not valid json: %v", err) + } + if event.FindingType != string(model.FindingTypeSubdomainTakeoverCNAME) { + t.Fatalf("unexpected finding type: %q", event.FindingType) + } + if event.DomainKey != scan.DomainKey { + t.Fatalf("unexpected domain key: %q", event.DomainKey) + } + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for finding to be published") + } + + // Mirrors main.go's shutdown sequence: stop the iterator so a blocked + // Next() call returns promptly, then cancel the context. + runtimeDeps.Iter.Stop() + cancelRun() + select { + case <-runDone: + case <-time.After(10 * time.Second): + t.Fatal("Run did not stop after context cancellation") + } +} + +func strPtrForTest(v string) *string { return &v } From 2638c1b381b5bbdffe9d9147866a4e506cab4ca5 Mon Sep 17 00:00:00 2001 From: lcampbell2 Date: Mon, 31 Aug 2026 11:48:28 -0300 Subject: [PATCH 65/65] run make tidy --- scanners/subdomain-takeover/go.mod | 2 +- scanners/subdomain-takeover/go.sum | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/scanners/subdomain-takeover/go.mod b/scanners/subdomain-takeover/go.mod index 5966eedfc8..4ab4251764 100644 --- a/scanners/subdomain-takeover/go.mod +++ b/scanners/subdomain-takeover/go.mod @@ -4,6 +4,7 @@ go 1.25.0 require ( github.com/kelseyhightower/envconfig v1.4.0 + github.com/nats-io/nats-server/v2 v2.14.5 github.com/nats-io/nats.go v1.52.0 github.com/rs/zerolog v1.35.1 ) @@ -16,7 +17,6 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/minio/highwayhash v1.0.4 // indirect github.com/nats-io/jwt/v2 v2.8.2 // indirect - github.com/nats-io/nats-server/v2 v2.14.5 // indirect github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect golang.org/x/crypto v0.55.0 // indirect diff --git a/scanners/subdomain-takeover/go.sum b/scanners/subdomain-takeover/go.sum index 47dc14f1fc..c72d763912 100644 --- a/scanners/subdomain-takeover/go.sum +++ b/scanners/subdomain-takeover/go.sum @@ -4,8 +4,6 @@ github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= @@ -20,22 +18,16 @@ github.com/nats-io/nats-server/v2 v2.14.5 h1:M6yeo/Xb7khi97RSEVELof3DForDqmYza3P github.com/nats-io/nats-server/v2 v2.14.5/go.mod h1:1D3iocrisKvWaD1B/imqarTqmaGrWMqALMLbEDo3v7Q= github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= -github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= -github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=