From cf022c5096bd9088b35f04748afd28fc8dc0b0cb Mon Sep 17 00:00:00 2001 From: Wong Ding Feng Date: Mon, 1 Jun 2026 00:10:21 +0800 Subject: [PATCH] feat: doc engine writes openapi.yaml and API.md from samples Co-Authored-By: Claude Sonnet 4.6 --- pyproject.toml | 1 + src/auto_reverse/doc/engine.py | 57 ++++++++++++++++++++++++++++++++++ tests/test_engine.py | 42 +++++++++++++++++++++++++ uv.lock | 28 +++++++++++++++++ 4 files changed, 128 insertions(+) create mode 100644 src/auto_reverse/doc/engine.py create mode 100644 tests/test_engine.py diff --git a/pyproject.toml b/pyproject.toml index 2fa2743..b2a5a94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "genson>=1.3.0", "mitmproxy>=12.2.3", "playwright>=1.60.0", + "pyyaml>=6.0.3", ] [project.scripts] diff --git a/src/auto_reverse/doc/engine.py b/src/auto_reverse/doc/engine.py new file mode 100644 index 0000000..993ab48 --- /dev/null +++ b/src/auto_reverse/doc/engine.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import threading +from typing import TYPE_CHECKING + +import yaml + +from auto_reverse.doc.markdown import render_markdown +from auto_reverse.doc.openapi import build_openapi +from auto_reverse.doc.schema import SchemaAccumulator + +if TYPE_CHECKING: + from pathlib import Path + + from auto_reverse.models import Signature + from auto_reverse.store import FlowStore + + +class DocEngine: + """Turn a new endpoint signature into inferred schemas + written outputs.""" + + def __init__( + self, store: FlowStore, out_dir: Path, title: str, use_llm: bool = True + ) -> None: + self._store = store + self._out = out_dir + self._title = title + self._use_llm = use_llm + self._lock = threading.Lock() + self._out.mkdir(parents=True, exist_ok=True) + + def document(self, sig: Signature) -> None: + record = self._store.get(sig) + if record is None: + return + req_acc = SchemaAccumulator() + resp_acc = SchemaAccumulator() + for flow in self._store.samples(sig): + rj = flow.request_json() + if rj is not None: + req_acc.add(rj) + sj = flow.response_json() + if sj is not None: + resp_acc.add(sj) + record.request_schema = req_acc.schema() + record.response_schema = resp_acc.schema() + if not record.summary: + record.summary = f"{sig.method} {sig.path_template}" + record.documented = True + self._write() + + def _write(self) -> None: + with self._lock: + records = self._store.endpoints() + spec = build_openapi(records, title=self._title) + (self._out / "openapi.yaml").write_text(yaml.safe_dump(spec, sort_keys=False)) + (self._out / "API.md").write_text(render_markdown(records, title=self._title)) diff --git a/tests/test_engine.py b/tests/test_engine.py new file mode 100644 index 0000000..f09a74c --- /dev/null +++ b/tests/test_engine.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from auto_reverse.doc.engine import DocEngine + +if TYPE_CHECKING: + from pathlib import Path +from auto_reverse.models import CapturedFlow +from auto_reverse.store import FlowStore, ScopeFilter + + +def _flow(path: str, resp: bytes) -> CapturedFlow: + return CapturedFlow( + method="GET", host="ex.com", path=path, query={}, req_headers={}, + req_body=None, status=200, + resp_headers={"content-type": "application/json"}, resp_body=resp, + timestamp=0.0, + ) + + +def test_engine_writes_spec_and_markdown(tmp_path: Path): + store = FlowStore(ScopeFilter(target_hosts={"ex.com"})) + engine = DocEngine(store, out_dir=tmp_path, title="ex.com API", use_llm=False) + store.ingest(_flow("/api/users", b'[{"id": 1}]')) + sig = store.endpoints()[0].signature + engine.document(sig) + spec = (tmp_path / "openapi.yaml").read_text() + assert "/api/users" in spec + assert (tmp_path / "API.md").read_text().startswith("# ex.com API") + + +def test_engine_infers_response_schema(tmp_path: Path): + store = FlowStore(ScopeFilter(target_hosts={"ex.com"})) + engine = DocEngine(store, out_dir=tmp_path, title="x", use_llm=False) + store.ingest(_flow("/api/users", b'[{"id": 1, "name": "Ada"}]')) + sig = store.endpoints()[0].signature + engine.document(sig) + rec = store.get(sig) + assert rec is not None + assert rec.response_schema is not None + assert rec.response_schema["type"] == "array" diff --git a/uv.lock b/uv.lock index 9f0ac89..93accff 100644 --- a/uv.lock +++ b/uv.lock @@ -134,6 +134,7 @@ dependencies = [ { name = "genson" }, { name = "mitmproxy" }, { name = "playwright" }, + { name = "pyyaml" }, ] [package.dev-dependencies] @@ -148,6 +149,7 @@ requires-dist = [ { name = "genson", specifier = ">=1.3.0" }, { name = "mitmproxy", specifier = ">=12.2.3" }, { name = "playwright", specifier = ">=1.60.0" }, + { name = "pyyaml", specifier = ">=6.0.3" }, ] [package.metadata.requires-dev] @@ -966,6 +968,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + [[package]] name = "ruamel-yaml" version = "0.19.1"