Native YAML decoding on the stock parser (WIP)

.github/docs/openapi2.txt

@@ -7,6 +7,13 @@ backwards-compatible with version 2, version 3 elements have been used.
 
 See https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md
 
+FUNCTIONS
+
+func UnmarshalFromData(data []byte, doc *T) error
+    UnmarshalFromData loads a document from swagger 2.0 bytes in either JSON or
+    YAML. The v3 side has Loader for this; here the whole job is the decode.
+
+
 TYPES
 
 type Header struct {

.github/workflows/fuzz.yml

@@ -0,0 +1,130 @@
+name: fuzz
+on:
+  pull_request:
+  push:
+    branches:
+    - master
+  schedule:
+  # Nightly, so a long run has a chance to reach code a 90s PR run never does.
+  - cron: '17 3 * * *'
+  workflow_dispatch:
+    inputs:
+      fuzztime:
+        description: 'How long to fuzz each target (go test -fuzztime syntax)'
+        required: false
+        default: 30m
+
+permissions:
+  contents: read
+
+jobs:
+  fuzz:
+    name: ${{ matrix.target }}
+    runs-on: ubuntu-latest
+    # A budget of 15m plus corpus minimization has to fit, and a hung target
+    # must not hold a runner forever.
+    timeout-minutes: 45
+    env:
+      # Keep in sync with the go workflow, whose cache this restores.
+      APISGURU_COMMIT: f7207cf0a5c56081d275ebae4cf615249323385d
+      GO111MODULE: 'on'
+      CGO_ENABLED: '0'
+    strategy:
+      # Every target reports independently: one crasher must not hide the rest.
+      fail-fast: false
+      matrix:
+        include:
+        - {package: ./openapi3, target: FuzzLoadFromData}
+        - {package: ./openapi3, target: FuzzLoadWithOrigins}
+        - {package: ./openapi3, target: FuzzYAMLMatchesJSON}
+        - {package: ./openapi3, target: FuzzSchemaVisitJSON}
+        - {package: ./openapi3, target: FuzzInternalizeRefs}
+        - {package: ./openapi3, target: FuzzInternalizeRefsKeepsMeaning}
+        - {package: ./openapi3, target: FuzzValidateOptions}
+        - {package: ./openapi3, target: FuzzInternalizeRefsCustomResolver}
+        - {package: ./openapi3, target: FuzzMarshalYAML}
+        - {package: ./openapi2, target: FuzzUnmarshalFromData}
+        - {package: ./internal/nodeindex, target: FuzzNew}
+        - {package: ./openapi2conv, target: FuzzToV3}
+        - {package: ./openapi3filter, target: FuzzValidateRequest}
+        - {package: ./openapi3filter, target: FuzzValidateResponse}
+        - {package: ./openapi3filter, target: FuzzBodyDecoder}
+        - {package: ./routers, target: FuzzRoutersAgree}
+    steps:
+
+    # Checked out first: setup-go keys its module and build cache on go.sum,
+    # and without one it caches nothing.
+    - uses: actions/checkout@v6
+
+    - uses: actions/setup-go@v6
+      with:
+        go-version: stable
+
+    # The APIs-guru corpus seeds the fuzzers alongside testdata. The go
+    # workflow caches it, so fetch it only when that cache is missing.
+    - name: APIs-guru corpus cache
+      id: apisguru
+      uses: actions/cache/restore@v4
+      with:
+        path: |
+          openapi2/testdata/APIs-guru-openapi-directory-*
+          openapi3/testdata/APIs-guru-openapi-directory-*
+        key: suite-${{ env.APISGURU_COMMIT }}
+
+    - name: Fetch APIs-guru corpus
+      if: steps.apisguru.outputs.cache-hit != 'true'
+      run: |
+        dir=openapi3/testdata/APIs-guru-openapi-directory-${APISGURU_COMMIT:0:7}
+        mkdir -p "$dir"
+        curl -fsSL "https://github.com/APIs-guru/openapi-directory/tarball/$APISGURU_COMMIT" | tar -xz -C "$dir"
+
+    # The generated corpus lives under $GOCACHE/fuzz and is what makes fuzzing
+    # cumulative: without it every run restarts from the seed corpus. The key
+    # carries the run id so each run writes a fresh entry, and restore-keys
+    # falls back to the most recent one for this target.
+    - name: Fuzzing corpus cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/go-build/fuzz
+        key: fuzz-${{ matrix.target }}-${{ github.run_id }}
+        restore-keys: fuzz-${{ matrix.target }}-
+
+    - name: Pick a budget
+      id: budget
+      run: |
+        if [[ -n '${{ inputs.fuzztime }}' ]]; then
+          fuzztime='${{ inputs.fuzztime }}'
+        elif [[ '${{ github.event_name }}' = schedule ]]; then
+          fuzztime=15m
+        else
+          fuzztime=90s
+        fi
+        echo "fuzztime=$fuzztime" >>"$GITHUB_OUTPUT"
+
+    # -run='^$' skips the ordinary tests: the seed corpus is already replayed by
+    # `go test ./...` in the go workflow, so this job only spends time fuzzing.
+    #
+    # The minimize budget is small on purpose. A minimize that cannot finish
+    # inside it writes the candidate it had and the job fails with "fuzzing
+    # process hung or terminated unexpectedly while minimizing", whose
+    # candidate then passes on replay: a red build and nothing to act on. The
+    # budget only decides how small the saved input gets, so spending less of
+    # it costs a larger seed and buys a signal that means what it says.
+    - name: Fuzz ${{ matrix.target }}
+      run: |
+        go test \
+          -run='^$' \
+          -fuzz='^${{ matrix.target }}$' \
+          -fuzztime='${{ steps.budget.outputs.fuzztime }}' \
+          -fuzzminimizetime=10s \
+          ${{ matrix.package }}
+
+    # A failing input is written next to the package it broke. Upload it so the
+    # crasher can be committed to the seed corpus, where it becomes a permanent
+    # regression test replayed by `go test ./...`.
+    - if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: crashers-${{ matrix.target }}
+        path: '**/testdata/fuzz/**'
+        if-no-files-found: ignore

.github/workflows/go.yml

@@ -25,17 +25,19 @@ jobs:
     name: ${{ matrix.go }} on ${{ matrix.os }}
     steps:
 
+    # Checked out first: setup-go keys its module and build cache on go.sum,
+    # and without one it caches nothing.
+    - uses: actions/checkout@v6
+
     - uses: actions/setup-go@v6
       with:
         go-version: ${{ matrix.go }}
 
     - if: runner.os == 'Linux'
       run: sudo apt install silversearcher-ag
 
-    - uses: actions/checkout@v6
-
     - name: Test suite fixtures cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: |
           openapi2/testdata/APIs-guru-openapi-directory-*
@@ -90,6 +92,9 @@ jobs:
     - run: go test -v -count=10 -run TestIssue741 -race ./...
       env:
         CGO_ENABLED: '1'
+    - run: go test -v -count=10 -run TestConcurrentDecodeDoesNotLeakOrigins -race ./...
+      env:
+        CGO_ENABLED: '1'
     - run: git --no-pager diff --exit-code
 
     - if: runner.os == 'Linux'
@@ -117,9 +122,9 @@ jobs:
         ! git grep -IErn '\s$'
 
     - if: runner.os == 'Linux'
-      name: Ensure use of unmarshal
+      name: Ensure yaml stays inside the decode layer
       run: |
-        [[ "$(git grep -F yaml. -- openapi3/ | grep -v _test.go | grep -v origin.go | wc -l)" = 2 ]]
+        [[ "$(git grep -F yaml. -- openapi3/ | grep -v _test.go | grep -vE 'openapi3/(origin|marsh|refs|loader|native_yaml.*|nativeyaml.*)[.](go|tmpl)' | wc -l)" -eq 0 ]]
 
     - if: runner.os == 'Linux'
       name: Use `loader := NewLoader(); loader.Load ...`
@@ -201,9 +206,9 @@ jobs:
     - if: runner.os == 'Linux'
       name: Ensure readableType() covers all possible values of resolved var
       run: |
-        [[ "$(git grep -F 'var resolved ' -- openapi3/loader.go | awk '{print $4}' | sort | tr '\n' ' ')" = "$RESOLVEDS" ]]
+        [[ "$(git grep -F 'var resolved ' -- openapi3/loader.go | awk '{print $4}' | sort | uniq | tr '\n' ' ')" = "$RESOLVEDS" ]]
       env:
-        RESOLVEDS: 'Callback CallbackRef ExampleRef HeaderRef LinkRef ParameterRef PathItem RequestBodyRef ResponseRef SchemaRef SecuritySchemeRef '
+        RESOLVEDS: 'CallbackRef ExampleRef HeaderRef LinkRef ParameterRef PathItem RequestBodyRef ResponseRef SchemaRef SecuritySchemeRef '
 
   check-goimports:
     runs-on: ubuntu-latest

Makefile

@@ -1,28 +1,41 @@
-.DEFAULT_GOAL := prepare
+.DEFAULT_GOAL := all
+.PHONY: help generate format test test-fast fuzz all
 
-.PHONY: help generate format test prepare
+# How long to spend on each fuzz target. Override: make fuzz FUZZTIME=10m
+FUZZTIME ?= 60s
 
-help:
-	@printf '%s\n' \
-		'Usage: make <target>' \
-		'' \
-		'  generate  Regenerate Go code, map helpers, and API documentation.' \
-		'  format    Apply the repository formatters and import cleanup.' \
-		'  test      Run the full Go package test suite.' \
-		'  prepare   Run generation, formatting, and tests in sequence.'
+## all: Run generation, formatting, and tests in sequence
+all: generate format test
 
+## generate: Regenerate Go code, map helpers, and API documentation
 generate:
 	go generate ./...
 	./maps.sh
 	./docs.sh
 
+## format: Apply the repository formatters and import cleanup
 format:
 	go fmt ./...
 
+## test: Run the full Go package test suite
 test:
 	go test ./...
 
-prepare:
-	$(MAKE) generate
-	$(MAKE) format
-	$(MAKE) test
+## test-fast: Run tests skipping the slow ones
+test-fast:
+	go test -short ./...
+
+## fuzz: Fuzz each target for FUZZTIME (default 60s)
+fuzz:
+	go test -run='^$$' -fuzz='^FuzzLoadFromData$$' -fuzztime=$(FUZZTIME) ./openapi3
+	go test -run='^$$' -fuzz='^FuzzLoadWithOrigins$$' -fuzztime=$(FUZZTIME) ./openapi3
+	go test -run='^$$' -fuzz='^FuzzYAMLMatchesJSON$$' -fuzztime=$(FUZZTIME) ./openapi3
+	go test -run='^$$' -fuzz='^FuzzSchemaVisitJSON$$' -fuzztime=$(FUZZTIME) ./openapi3
+	go test -run='^$$' -fuzz='^FuzzUnmarshalFromData$$' -fuzztime=$(FUZZTIME) ./openapi2
+	go test -run='^$$' -fuzz='^FuzzNew$$' -fuzztime=$(FUZZTIME) ./internal/nodeindex
+	go test -run='^$$' -fuzz='^FuzzToV3$$' -fuzztime=$(FUZZTIME) ./openapi2conv
+	go test -run='^$$' -fuzz='^FuzzValidateRequest$$' -fuzztime=$(FUZZTIME) ./openapi3filter
+
+## help: list targets
+help:
+	@grep -E '^## ' $(MAKEFILE_LIST) | sed 's/## /  make /'