diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 000000000..be2e96ba1 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,15 @@ +{ + "name": "linkeddatahub", + "description": "Skills for building Linked Data dataspaces, datasets and applications on LinkedDataHub", + "owner": { + "name": "AtomGraph", + "url": "https://atomgraph.com" + }, + "plugins": [ + { + "name": "ldh", + "source": "./plugins/ldh", + "description": "Build Linked Data dataspaces, datasets and applications on LinkedDataHub with the ldh CLI" + } + ] +} diff --git a/.env_sample b/.env_sample index 6738d3a56..591bd0679 100644 --- a/.env_sample +++ b/.env_sample @@ -17,3 +17,8 @@ OWNER_STATE_OR_PROVINCE=Denmark OWNER_COUNTRY_NAME=DK MAX_CONTENT_LENGTH=2097152 + +# the assistant's plan service image tag (ghcr.io/atomgraph/webalgebra-server-ldh); `local` for an image built from a REST-VKG checkout +# WEBALGEBRA_VERSION=latest +# the OpenAI model the assistant plans and writes queries with; gpt-4o when unset +# OPENAI_MODEL=gpt-4o diff --git a/.github/workflows/load-tests.yml b/.github/workflows/load-tests.yml index 92756ffe6..5277b48a5 100644 --- a/.github/workflows/load-tests.yml +++ b/.github/workflows/load-tests.yml @@ -12,6 +12,23 @@ jobs: - name: Add bin/ and its subdirectories to PATH run: | find "$GITHUB_WORKSPACE/bin" -type d >> "$GITHUB_PATH" + # the SPARQL load tests make the end-user dataspace public first (ldh admin make-public), the way a + # real deployment does - anonymous /sparql is what they put under load. That needs the ldh CLI, built + # here from this checkout exactly as the HTTP-tests workflow does (JDK 21, then the sibling library + # cli/ resolves by version, then the CLI). + - name: Set up Java 21 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '21' + - name: Build the linkeddatahub-rdf library + run: mvn -B install + working-directory: rdf + - name: Build the ldh CLI + run: mvn -B package + working-directory: cli + - name: Add the ldh launcher to PATH + run: echo "$GITHUB_WORKSPACE/cli/bin" >> "$GITHUB_PATH" - name: Generating server certificate run: | server-cert-gen.sh .env nginx ssl @@ -28,10 +45,27 @@ jobs: run: docker compose -f docker-compose.yml -f ./tests/http/docker-compose.http-tests.yml -f ./tests/load/docker-compose.load-tests.yml --env-file ./tests/http/.env up --build -d - name: Wait for the server to start... run: while ! (status=$(curl -k -s -w "%{http_code}\n" https://localhost:4443 -o /dev/null) && echo "$status" && echo "$status" | grep "403") ; do sleep 1 ; done # wait for the webapp to start (returns 403 by default) + - name: Make the end-user dataspace public + # the platform writes the owner keystore into the bind-mounted ssl/owner on first boot; the suite's + # SPARQL tests send anonymous queries, which only reach the store once the dataspace is public + run: | + ldh admin make-public \ + -c tests/http/ssl/owner/keystore.p12 \ + -p "$(cat secrets/owner_cert_password.txt)" \ + https://localhost:4443/ - name: Run load test scripts run: ./run.sh shell: bash working-directory: tests/load + env: + # the same stack the steps above brought up (project name from tests/http/.env); the SPARQL tests + # drive the loadgen containers and inspect fuseki through this + COMPOSE_CMD: >- + docker compose + -f ${{ github.workspace }}/docker-compose.yml + -f ${{ github.workspace }}/tests/http/docker-compose.http-tests.yml + -f ${{ github.workspace }}/tests/load/docker-compose.load-tests.yml + --env-file ${{ github.workspace }}/tests/http/.env - name: Generate test summary if: always() run: python3 scripts/generate_test_summary.py tests/load/out tests/load/out/report.md diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 385670d70..7480507ff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -161,6 +161,13 @@ jobs: # strip the literal prefix echo "FULL_VERSION=${RAW_REF#linkeddatahub-}" >> $GITHUB_ENV + # The CLI resolves linkeddatahub-rdf at its own version, which the tagged checkout holds. Maven + # Central does not: release.sh deploys the library after pushing the tag, and Central publishes + # it minutes later, while this job runs seconds after the push. + - name: Build the linkeddatahub-rdf library + run: mvn -B install + working-directory: rdf + # release.sh keeps cli/pom.xml at the platform version, so the tagged commit already carries it # and the jar manifest ldh --version reads gets it from there - name: Build the ldh CLI diff --git a/AGENTS.md b/AGENTS.md index d0d322beb..0ae9e0ee1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,6 +73,12 @@ The dataspace exposes a **read-only SPARQL 1.1 Query** endpoint (advertised via Write portable, standard SPARQL: use explicit `GRAPH` patterns, no engine-specific extensions. +**The default graph is empty.** Every document is a named graph whose name is the document's URL, so a triple pattern outside `GRAPH` matches nothing. Put the patterns inside `GRAPH ?g { ... }` to query across every document, or inside `GRAPH { ... }` for one document; a subquery needs its own `GRAPH` as well. A query that returns no rows without a `GRAPH` clause is not evidence that the data is absent. + +**One document per resource, so a join crosses graphs.** A resource is described in the document about it (`foaf:primaryTopic`), and the resources it links to - an order's customer, a product's category, an employee's manager - are each described in their own document, in another graph. A pattern that follows such a link therefore needs a `GRAPH` of its own for each resource: `GRAPH ?o { ?order schema:customer ?customer } GRAPH ?c { ?customer schema:legalName ?name }`, never both triples in one `GRAPH ?g`, which asks for them in the same document and matches nothing. Only a resource's own companions - its address, its contact point, an order's line items - share its document. When in doubt, give every subject variable its own graph variable: it costs nothing when two resources do share a document, and it is the only pattern that works when they do not. + +**A property is used on instances of its domain.** The ontology (`Link rel=lds:ontology`) declares each property's `rdfs:domain` and `rdfs:range`, and the data follows it: a property declared on `schema:Person` is not on the `schema:Corporation` that employs the person. To reach it, follow the ontology's path - the corporation's `schema:employee`, then the person's `schema:address`. + Outbound `SERVICE` and `LOAD` — from the triplestore and from the platform alike — are routed through the `egress` forward proxy, which refuses loopback, private and link-local destinations. A federated query reaches public endpoints and cannot reach the deployment's own services. With no proxy configured and `ALLOW_INTERNAL_URLS` unset, in-JVM `SERVICE` (in a `PATCH` update or an import mapping) is disabled outright. ## Content & document model diff --git a/CHANGELOG.md b/CHANGELOG.md index dd28de1de..46b383324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,44 @@ +## [Unreleased] +### Migration +- Top-level LDH stylesheet params and variables carry the `ldh:` prefix; overrides of the unprefixed names are ignored +- Shared stylesheets query through `ldh:query-result()`; `ac:value-intersect()`/`ac:value-except()` are removed +- **BREAKING**: the `$lds:Context` stylesheet param is removed, as it exposed service credentials to stylesheets; use `lds:context()` +- Existing dataspaces need an authorization granting public read access to `/context` on both origins (new ones get it seeded) +- **BREAKING**: `ldh import csv|rdf` print the URI of the created import instead of the target document's +- **BREAKING**: `ldh --proxy` keeps the request's `Host`; pass the plain proxy instead of an `admin.`-prefixed one + +### Added +- `/context` serves the public description of every dataspace, and end-user responses link their admin application (`lds:admin`) +- `ldh admin` commands take the end-user base URI +- `ldh admin import ontology --file` installs the namespace ontology from a local file in one step +- `ldh import csv|rdf --wait` waits for the import to finish +- An assistant that turns a natural-language request into a Web-Algebra plan, runs it on Execute and shows the result as a view +- Assistant conversations are stored as `ldh:Chat` blocks in the document +- A `web-algebra` service in the stack, acting on the reader's behalf; the assistant writes plans only when the optional `openai_api_key` secret is declared +- The agent guide is served at `/AGENTS.md` on every dataspace +- An `ldh` Claude Code plugin with skills for building dataspaces, datasets, apps and packages with the CLI +- A `GET` on `/sparql` without a query returns the endpoint's SPARQL Service Description + +### Changed +- An external HTML page with no RDF representation opens in a new tab; the proxy answers `406` instead of `502` +- Every chart, map and 3D graph canvas is initialised by one template, wherever it is rendered +- External `/sparql` requests are capped per client and in total by nginx (`NGINX_SPARQL_*`), answering `429` beyond the limits +- Fuseki restarts on an out-of-memory error instead of running on broken +- Web-Client dependency bumped to 6.0.6 + +### Fixed +- A malformed `Accept`-family header no longer turns an error response into a `500`; an invalid `accept` parameter is refused with `400` +- Proxied responses always vary on `Accept`, so a cached RDF response is no longer served to a browser navigation +- The proxy serves the boolean result of an upstream `ASK` query +- A document imported under another name than its ontology's no longer takes the dataspace down +- LDH's ontologies import SPIN, SP and OWL by the IRIs those ontologies declare +- Nested property lists of blank nodes lay out for their own cell instead of collapsing to one word per line +- Charts no longer fail on datetimes with fractional milliseconds +- The version history link announces that it opens a dialog +- `rdf:XMLLiteral` values are stored as Exclusive XML Canonicalization again +- A block saved through its edit form labels its resource values instead of showing the URI's fragment +- An ontology-defined view no longer flashes its own metadata before its results + ## [6.1.0] - 2026-10-05 ### Migration - **BREAKING**: `ldh` drops `-b`/`--base`; dataspace-level commands take the base URI as their positional, defaulting to `LDH_BASE` diff --git a/CLAUDE.md b/CLAUDE.md index c2e17250f..eabf8d48d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -41,7 +41,7 @@ make drop # Complete reset (down -v, then wip make rdf # Install the linkeddatahub-rdf library make cli # Build the ldh CLI (installs rdf first), print the PATH export -make cli-version # Set rdf/ and cli/ poms to the platform version in pom.xml +make cli-version # Set the rdf/ and cli/ poms and the ldh plugin to the platform version in pom.xml ``` ### Testing @@ -102,12 +102,23 @@ Configuration is split across two files: Multiple dataspaces can share the same backend SPARQL service. +The admin pairing is the `admin.` subdomain and nothing else: the server derives it +(`EndUserDataspaceImpl.getAdminDataspace()`) and publishes it, so no client derives it again. An +end-user response carries `Link: ; rel="…/dataspaces#admin"` (the CLI's admin commands +follow it), and `/context` serves `PublicContext`: every dataspace's description (type, title, +origin, ontology, stylesheet, `lds:public`) plus `lds:admin`, readable by anyone through a seeded +authorization. Stylesheets read it through `lds:context()`: the server twin dereferences `/context` +(same-site, so `SameSiteSourceResolver` fetches it), the client twin reads the copy the bootstrap +preloads. No stylesheet param carries context: the full context dataset also holds the service +endpoints and credentials from `system.trig` and `secrets/credentials`, and stays in Java. + ### Service Architecture The application runs as a multi-container setup: - **nginx**: Reverse proxy and SSL termination (wildcard subdomain routing for dataspaces) - **linkeddatahub**: Main Java application (Tomcat) - **fuseki**: One SPARQL server holding a TDB2 dataset per dataspace role (`config/fuseki/config.ttl`), named after the dataspace origin (deployment host dropped, role appended: `end-user`, `admin`, `northwind-traders.demo.end-user`, …), each under `fuseki//`; bound to apps in `config/system.trig` - **egress**: Squid forward proxy for the store's and platform's outbound requests (SPARQL `SERVICE`, `LOAD`): public destinations only, so a query cannot reach another dataset, Varnish or the platform +- **web-algebra**: REST-VKG's `webalgebra-server-ldh` image — executes Web-Algebra plans and writes them from natural language for the assistant drawer (`xsl/client/chat.xsl`). nginx forwards `/webalgebra` (a reserved path, like `/static/` and `/uploads/`) to it with the caller's certificate; it presents the secretary's certificate to this instance with `On-Behalf-Of` naming the caller, which `WebIDFilter` honours because every agent's WebID document carries ` acl:delegates `. It addresses the instance by its public origin, rewritten to `nginx:9443` (`WEBALGEBRA_PROXY_HOST`), the same trick the platform's own `ClientUriRewriteFilter` plays - **varnish-frontend/varnish-admin/varnish-end-user**: Caching layers (admin and end-user caches both front the single `fuseki`) ### Data Flow @@ -162,6 +173,12 @@ ldh create container --parent "$LDH_BASE" --title "Some" --slug some ldh admin add agent --agent "$AGENT_URI" "${ADMIN_BASE}acl/groups/writers/" ``` +The `ldh` Claude Code plugin (`plugins/ldh/`, published by the marketplace in `.claude-plugin/`) +teaches agents to build on LinkedDataHub with this CLI: one skill each for provisioning a dataspace, +modelling and importing a dataset, building an app, and writing a package. Its skills quote command +options and config shapes, so a change to either updates them in the same commit. `plugin.json` +carries the platform version, set by `release.sh` and `make cli-version` along with the poms. + ## The RDF library `rdf/` builds `com.atomgraph:linkeddatahub-rdf` — the vocabularies (`com.atomgraph.linkeddatahub.rdf.vocabulary`) @@ -200,6 +217,12 @@ by extension. Commands that create or append to a document print its URL as the stdout (diagnostics go to stderr), so `item=$(ldh create item ...)` works; exit codes are `0` success, `1` HTTP or runtime failure, `2` usage error. +The admin commands take the end-user base and find the admin application from its `lds:admin` link, +so scripts never prefix `admin.` themselves. `--proxy` connects to the proxy but keeps the logical +`Host`, which is how nginx routes dataspaces. `admin import ontology --file ns.ttl` installs the +namespace ontology in one step, and `import csv|rdf` print the import's URI, with `--wait` to block +until it ends. + Packages are declarative: an application imports one with a single ` ldh:import ` triple in its settings, and `ldh packages list`, `ldh packages add` and `ldh packages remove` write that triple through `PATCH /settings`. `packages list` reads the diff --git a/Dockerfile b/Dockerfile index a95246e1f..35072ea33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,9 @@ RUN mvn -B -Pstandalone dependency:go-offline COPY src /usr/src/platform/src +# the agent guide the WAR serves at /AGENTS.md (pom.xml, maven-war-plugin webResources) +COPY AGENTS.md /usr/src/platform/AGENTS.md + RUN mvn -Pstandalone clean install # ============================== diff --git a/README.md b/README.md index c6a90abbd..011ccf09a 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ The [`ldh` command line interface](#command-line-interface) is attached to every - `secrets/client_truststore_password.txt` - `secrets/owner_cert_password.txt` - `secrets/secretary_cert_password.txt` + - `secrets/openai_api_key.txt` — optional: an OpenAI API key for the assistant drawer, declared in `docker-compose.override.yml` like the OAuth secrets; without it the assistant runs plans but cannot write them The one you will need to remember in order to authenticate with LinkedDataHub using WebID client certificate is `owner_cert_password`. 5. Launch the application services by running this from command line: ```shell @@ -217,6 +218,8 @@ _:warning: Do not use blank nodes to identify applications or services. We recom
Password of the secretary's WebID certificate
client_truststore_password
Password of the client truststore
+
openai_api_key
+
OpenAI API key the web-algebra service writes the assistant's plans with. Optional, declared in the override together with OPENAI_API_KEY_FILE; the service executes plans without it, and the drawer then reports that a plan could not be written
google_client_id
Google's OAuth client ID
Login with Google authentication is enabled when this value is provided
@@ -359,7 +362,6 @@ LinkedDataHub includes an HTTP [test suite](https://github.com/AtomGraph/LinkedD * [SPARQL.js](https://github.com/RubenVerborgh/SPARQL.js) * [OpenLayers](https://openlayers.org) * [Google Charts](https://developers.google.com/chart) -* [xml-c14n-sync](https://github.com/AtomGraph/xml-c14n-sync) ### Java diff --git a/cli/README.md b/cli/README.md index 5fa75145f..1bf231d3a 100644 --- a/cli/README.md +++ b/cli/README.md @@ -118,12 +118,28 @@ The commands that address no document of their own take the dataspace's base URI argument instead, where every other command puts its target: the `packages` family, `admin clear ontology`, `admin create ontology`, `admin create group`, `admin create authorization`, `admin import ontology` and `admin make-public`. `LDH_BASE` is its default, so with it exported the -argument is left out: +argument is left out. + +The admin commands take the end-user base too. They find the admin application from its +`lds:admin` `Link` header, so a script never derives the admin URI; a base without that link, such as +the admin base itself, is used as given: ```bash ldh packages add --package https://packages.linkeddatahub.com/editor/taxonomy/#this https://localhost:4443/ -ldh admin clear ontology https://admin.localhost:4443/ -ldh admin create group --name Editors --member "$AGENT_URI" https://admin.localhost:4443/ +ldh admin clear ontology https://localhost:4443/ +ldh admin create group --name Editors --member "$AGENT_URI" https://localhost:4443/ +``` + +`admin import ontology --file` installs a local file as the dataspace's namespace ontology in one step: +it takes the ontology IRI from the end-user base's `lds:ontology` link and resolves the file against it +(so `@prefix : <#>` names the ontology), replaces the `ontologies/namespace/` document keeping its own +description and seeded header (with the `owl:imports` of the platform's ontology), and clears the +ontology cache. A re-run converges, and a term removed from the file is gone. `--file -` reads stdin +with `-t/--content-type`. `--source` still imports an external ontology by URI with derived +constructors. + +```bash +ldh admin import ontology --file admin/model/ns.ttl ``` ```bash @@ -150,15 +166,22 @@ without them (the taxonomy editor package rejects a concept with no `skos:inSche - Commands that create or append to a document print its URL as the only line on stdout, so shell pipelines keep working: `item=$(ldh create item ...)`. `add file` prints the content-addressed - upload URI (`uploads/{sha1}` under the root of the document's origin). All diagnostics go to stderr. + upload URI (`uploads/{sha1}` under the root of the document's origin). `import csv` and + `import rdf` print the URI of the import they create, a fragment of the target document + (`${import%%#*}` is the document). All diagnostics go to stderr. +- An import runs in the background on the server. `import csv|rdf --wait` waits until it has ended + (`prov:endedAtTime`), prints its triple count to stderr, and exits `1` with the error message if it + failed, or after `--timeout` seconds (default 1200). - `push` writes many documents in one run and prints one line per written document URL or upload URI, in write order, so the listing greps and cuts like `packages list` does. Progress (`PUT <- `, `POST <- `) and `Skipping ` lines go to stderr. On the first failed request the run stops with exit code `1`; stdout holds what was written before it. - Exit codes: `0` success, `1` HTTP error status or runtime failure (message on stderr, stack trace with `--verbose`), `2` usage error. -- `--proxy` rewrites the request URI's origin to the proxy's origin, like the scripts do; printed - URLs keep the logical origin. +- `--proxy` sends every request to the proxy's scheme, host and port but keeps the logical URI's + path and `Host` header, so the platform still routes the dataspace (an admin request through the + end-user proxy reaches the admin application) and answers with logical URIs, which are the ones + printed. No admin proxy is derived. - `post`/`put` read RDF from an optional `FILE` argument, recognizing the syntax from its extension (`ldh put "$LDH_BASE" root.ttl`), or from stdin, where `-t/--content-type` is required since a stream carries no name; either way relative URIs resolve against the target @@ -332,11 +355,16 @@ shell scripts. - `add generic-service` drops the documented-but-unparsed `--slug` option. - `add csv-import`/`import csv` default `--delimiter` to `,` (the script required it despite documenting a default). -- `import csv`/`import rdf` run their steps in-process instead of spawning subscripts, and pass - `--description` through to the import metadata. +- `import csv`/`import rdf` run their steps in-process instead of spawning subscripts, pass + `--description` through to the import metadata, print the import's URI instead of the target's, and + add `--wait`. +- The admin commands accept the end-user base and follow its `lds:admin` link; the scripts derived the + admin base and proxy by prefixing `admin.` to the host. `--proxy` keeps the `Host` header instead of + rewriting it. - `admin import ontology` reads the `construct-constructors` query text by dereferencing its document instead of going through a `SELECT` on `/sparql`; the CONSTRUCT it then runs over the - scratch graph is unchanged. + scratch graph is unchanged. It adds `--file`, which has no script equivalent: the apps scripted the + same install as a reset `PATCH`, a `POST` with an `@base` and a cache clear. - `create item`/`create container` add `--primary-topic`, which the scripts have no equivalent for: they could only create a document that says nothing about what it is about, leaving a `patch` as the only way to assert it. diff --git a/cli/pom.xml b/cli/pom.xml index d78a8ae3d..4bed16a72 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -4,7 +4,7 @@ com.atomgraph linkeddatahub-cli - 6.1.0 + 6.2.0-SNAPSHOT jar LinkedDataHub CLI diff --git a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/BaseCommand.java b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/BaseCommand.java index 19e4a0f50..478c599b1 100644 --- a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/BaseCommand.java +++ b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/BaseCommand.java @@ -20,15 +20,19 @@ import com.atomgraph.linkeddatahub.cli.http.ClientFactory; import com.atomgraph.linkeddatahub.cli.http.HttpException; import com.atomgraph.linkeddatahub.cli.http.LDHClient; +import com.atomgraph.linkeddatahub.cli.http.Links; import com.atomgraph.linkeddatahub.cli.mixin.CertAuthMixin; import com.atomgraph.linkeddatahub.cli.mixin.ProxyMixin; +import com.atomgraph.linkeddatahub.rdf.vocabulary.LDS; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import java.util.concurrent.Callable; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; @@ -82,21 +86,40 @@ protected LDHClient getClient() { getCertAuth().validate(getSpec()); client = new LDHClient(ClientFactory.createClient(getCertAuth().getCertFile(), getCertAuth().getCertPassword()), - new MediaTypes(), getEffectiveProxy()); + new MediaTypes(), getProxyMixin().getProxy()); } return client; } /** - * Returns the proxy URI applied to requests. Commands that target the admin application - * override this to convert the proxy to the admin subdomain. + * Returns the targets of a resource's Link response headers with the given relation, read with HEAD. * - * @return proxy URI or null + * @param uri resource URI + * @param rel relation URI + * @return link targets, possibly empty */ - protected URI getEffectiveProxy() + protected List getLinkTargets(URI uri, String rel) { - return getProxyMixin().getProxy(); + try (Response response = HttpException.check(uri, getClient().head(uri, ACCEPT_TURTLE))) + { + return Links.targets(response, rel); + } + } + + /** + * Returns the base URI of the administration application of a dataspace. + * An end-user dataspace advertises it as lds:admin; a base without that link is taken to be the admin + * base already, so callers may pass either. + * + * @param base end-user or admin base URI + * @return admin base URI + */ + protected URI getAdminBase(URI base) + { + List admin = getLinkTargets(base, LDS.admin.getURI()); + + return admin.isEmpty() ? base : admin.get(0); } /** diff --git a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/Get.java b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/Get.java index 71b1fd469..7601ecb16 100644 --- a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/Get.java +++ b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/Get.java @@ -19,7 +19,6 @@ import com.atomgraph.linkeddatahub.cli.BaseCommand; import com.atomgraph.linkeddatahub.cli.http.HttpException; import com.atomgraph.linkeddatahub.cli.util.Mementos; -import com.atomgraph.linkeddatahub.cli.util.URIRewriter; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.MultivaluedHashMap; import jakarta.ws.rs.core.MultivaluedMap; @@ -107,14 +106,14 @@ else if (timeGate) * Returns the URI of the Memento a TimeGate redirected to. * * @param response TimeGate response - * @return Memento URI, with the logical origin restored when the request went through a proxy + * @return Memento URI; a proxied request keeps its logical Host, so the platform answers with the logical origin */ private URI mementoURI(Response response) { URI location = response.getLocation(); if (location == null) throw new IllegalStateException("TimeGate <" + requestURI() + "> did not redirect to a version of <" + target + ">"); - return getEffectiveProxy() != null ? URIRewriter.rewrite(location, target) : location; + return location; } /** diff --git a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/ClearOntology.java b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/ClearOntology.java index c28f43061..e999c574a 100644 --- a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/ClearOntology.java +++ b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/ClearOntology.java @@ -33,7 +33,8 @@ * next request already reads the new version. Without it nothing is reloaded and the closures rebuild * lazily, which is what a caller wanting only a cold cache should ask for. * - * The base URI is the base of the admin application. + * The base URI is the dataspace's, end-user or admin: an end-user base is followed to its admin application + * through its lds:admin link. * * @author Martynas Jusevičius {@literal } */ @@ -50,7 +51,7 @@ public class ClearOntology extends BaseCommand @Override public Integer call() throws Exception { - URI base = baseMixin.require(getSpec()); + URI base = getAdminBase(baseMixin.require(getSpec())); URI target = URI.create(base + "clear"); Form form = ontology != null ? new Form("uri", ontology.toString()) : new Form(); diff --git a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/acl/CreateAuthorization.java b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/acl/CreateAuthorization.java index 1fabd986b..1f0c496ee 100644 --- a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/acl/CreateAuthorization.java +++ b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/acl/CreateAuthorization.java @@ -93,7 +93,7 @@ public Integer call() throws Exception if (!append && !control && !read && !write) throw new ParameterException(getSpec().commandLine(), "At least one of '--append', '--control', '--read', '--write' is required"); - URI doc = URIs.childURI(URI.create(base + "acl/authorizations/"), slug != null ? slug : Slugs.defaultSlug()); + URI doc = URIs.childURI(URI.create(getAdminBase(base) + "acl/authorizations/"), slug != null ? slug : Slugs.defaultSlug()); List modes = new ArrayList<>(); if (append) modes.add(ACL.Append); diff --git a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/acl/CreateGroup.java b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/acl/CreateGroup.java index a6988a66a..5b86d4ca5 100644 --- a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/acl/CreateGroup.java +++ b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/acl/CreateGroup.java @@ -57,7 +57,7 @@ public class CreateGroup extends BaseCommand @Override public Integer call() throws Exception { - URI base = baseMixin.require(getSpec()); + URI base = getAdminBase(baseMixin.require(getSpec())); URI doc = URIs.childURI(URI.create(base + "acl/groups/"), slug != null ? slug : Slugs.defaultSlug()); put(getClient(), doc, Acl.group(doc, uri, name, description, members)); diff --git a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/acl/MakePublic.java b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/acl/MakePublic.java index 86f76da27..ae2641b92 100644 --- a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/acl/MakePublic.java +++ b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/acl/MakePublic.java @@ -19,7 +19,6 @@ import com.atomgraph.linkeddatahub.cli.BaseCommand; import com.atomgraph.linkeddatahub.cli.http.HttpException; import com.atomgraph.linkeddatahub.cli.mixin.BaseMixin; -import com.atomgraph.linkeddatahub.rdf.URIs; import com.atomgraph.linkeddatahub.rdf.Updates; import java.net.URI; import picocli.CommandLine.Command; @@ -41,7 +40,7 @@ public class MakePublic extends BaseCommand public Integer call() throws Exception { URI base = baseMixin.require(getSpec()); - URI adminBase = URIs.adminBase(base); + URI adminBase = getAdminBase(base); URI target = URI.create(adminBase + "acl/authorizations/public/"); HttpException.check(target, getClient().patch(target, Updates.makePublic(base, adminBase))).close(); @@ -49,11 +48,4 @@ public Integer call() throws Exception return 0; } - @Override - protected URI getEffectiveProxy() - { - // the request targets the admin app, so the proxy origin gets the admin subdomain too - return getProxyMixin().getProxy() != null ? URIs.adminBase(getProxyMixin().getProxy()) : null; - } - } diff --git a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/ontologies/CreateOntology.java b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/ontologies/CreateOntology.java index 0431d906b..4fb342433 100644 --- a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/ontologies/CreateOntology.java +++ b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/ontologies/CreateOntology.java @@ -53,7 +53,7 @@ public class CreateOntology extends BaseCommand @Override public Integer call() throws Exception { - URI base = baseMixin.require(getSpec()); + URI base = getAdminBase(baseMixin.require(getSpec())); URI doc = URIs.childURI(URI.create(base + "ontologies/"), slug != null ? slug : Slugs.defaultSlug()); put(getClient(), doc, Ontologies.ontology(doc, uri, label, comment)); diff --git a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/ontologies/ImportOntology.java b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/ontologies/ImportOntology.java index b187dda50..79ccc22b5 100644 --- a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/ontologies/ImportOntology.java +++ b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/admin/ontologies/ImportOntology.java @@ -20,6 +20,7 @@ import com.atomgraph.linkeddatahub.cli.http.HttpException; import com.atomgraph.linkeddatahub.cli.mixin.BaseMixin; import com.atomgraph.linkeddatahub.rdf.Ontologies; +import com.atomgraph.linkeddatahub.rdf.vocabulary.LDS; import com.atomgraph.linkeddatahub.rdf.vocabulary.SP; import jakarta.ws.rs.core.Form; import jakarta.ws.rs.core.MediaType; @@ -27,62 +28,173 @@ import java.net.URI; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.RDFNode; import org.apache.jena.rdf.model.Resource; +import org.apache.jena.sparql.vocabulary.FOAF; +import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; +import picocli.CommandLine.ParameterException; /** - * Imports an external ontology into a document: the vocabulary itself, the class constructors derived - * from it, and a foaf:primaryTopic naming what the document is about. - * Mirrors bin/admin/ontologies/import-ontology.sh. - * - * The vocabulary is fetched through the Linked Data proxy and stays in the target, which is what makes - * the import an import: the graph carries the vocabulary's own header, so resolving its URI finds this - * document with the derived annotations on it. The document remains a dh:Item and never - * claims to be the ontology. The same shape the browser's import produces, and the one a materialized - * package ontology has. - * - * The base URI is the base of the admin application. + * Imports an ontology into a document, either an external one by URI or a local file. + *

+ * With --source, imports an external ontology: the vocabulary itself, the class constructors derived + * from it, and a foaf:primaryTopic naming what the document is about. Mirrors + * bin/admin/ontologies/import-ontology.sh. The vocabulary is fetched through the Linked Data proxy and + * stays in the target, which is what makes the import an import: the graph carries the vocabulary's own header, so + * resolving its URI finds this document with the derived annotations on it. The document remains a + * dh:Item and never claims to be the ontology. The same shape the browser's import produces, and the one + * a materialized package ontology has. + *

+ * With --file, installs a local file as the dataspace's namespace ontology: the end-user base's + * lds:ontology link names the ontology, the file is parsed against it, and the namespace document is + * replaced, keeping its own description and its ontology header (with the owl:imports of the platform's + * ontology), so that a re-run converges and a term removed from the file is gone. The cached ontology is then cleared. + * No constructors are derived: the file declares its own. + *

+ * The base URI is the dataspace's. --source accepts the end-user or the admin base; --file + * needs the end-user one, which links both the ontology and the admin application. * * @author Martynas Jusevičius {@literal } */ -@Command(name = "ontology", description = "Imports an external ontology into a document: the vocabulary, the class constructors derived from it, and a foaf:primaryTopic of the source.") +@Command(name = "ontology", description = "Imports an ontology into a document: an external one (--source) with the class constructors derived from it, or a local file (--file) as the dataspace's namespace ontology.") public class ImportOntology extends BaseCommand { - /** Accepted response media type of the proxied vocabulary and the derived constructors */ private static final MediaType[] ACCEPT_RDF_XML = { com.atomgraph.core.MediaType.APPLICATION_RDF_XML_TYPE }; - /** Path of the document holding the constructor derivation query, relative to the admin base URI */ private static final String CONSTRUCT_CONSTRUCTORS_PATH = "queries/construct-constructors/"; + /** Path of the namespace ontology document, relative to the admin base: the slug the platform seeds */ + private static final String NAMESPACE_DOCUMENT_PATH = "ontologies/namespace/"; @Mixin private BaseMixin baseMixin; - @Option(names = "--source", required = true, paramLabel = "SOURCE_URI", description = "URI of the imported ontology") - private URI source; + @ArgGroup(exclusive = true, multiplicity = "1") + private Input input; - @Option(names = "--graph", required = true, paramLabel = "GRAPH_URI", description = "URI of the document the ontology is imported into") + @Option(names = "--graph", paramLabel = "GRAPH_URI", description = "URI of the document the ontology is imported into (required with --source; default with --file: the namespace ontology document)") private URI graph; + @Option(names = { "-t", "--content-type" }, paramLabel = "MEDIA_TYPE", description = "Media type of the --file (default: from the file extension; required for stdin)") + private String contentType; + + /** What is imported: an external ontology or a local file. */ + static class Input + { + + @Option(names = "--source", required = true, paramLabel = "SOURCE_URI", description = "URI of the imported external ontology") + private URI source; + + @Option(names = "--file", required = true, paramLabel = "PATH", description = "Path to a local ontology file to install as the namespace ontology ('-' reads stdin)") + private String file; + + } + @Override public Integer call() throws Exception { - URI base = baseMixin.require(getSpec()); - Model vocabulary = getVocabulary(base, source); - String query = getConstructorQuery(base); + if (input.source != null) return importSource(baseMixin.require(getSpec())); + + return importFile(baseMixin.require(getSpec())); + } + + /** + * Imports an external ontology with its derived constructors. + * + * @param base dataspace base URI + * @return exit code + */ + protected Integer importSource(URI base) + { + if (graph == null) throw new ParameterException(getSpec().commandLine(), "Missing required option: '--graph=GRAPH_URI' (required with --source)"); + + URI adminBase = getAdminBase(base); + Model vocabulary = getVocabulary(adminBase, input.source); + String query = getConstructorQuery(adminBase); // the vocabulary goes into the target first, so the CONSTRUCT has it to read and it stays afterwards post(getClient(), graph, vocabulary); - post(getClient(), graph, construct(base, query, graph)); - post(getClient(), graph, Ontologies.annotation(graph, source)); + post(getClient(), graph, construct(adminBase, query, graph)); + post(getClient(), graph, Ontologies.annotation(graph, input.source)); print(graph); return 0; } + /** + * Installs a local file as the dataspace's namespace ontology. + * + * @param base end-user dataspace base URI + * @return exit code + * @throws java.io.IOException if the file cannot be read + */ + protected Integer importFile(URI base) throws java.io.IOException + { + Path path = input.file.equals("-") ? null : Path.of(input.file); + String mediaType = resolveContentType(contentType, path); + + URI adminBase = getAdminBase(base); + if (adminBase.equals(base)) throw new ParameterException(getSpec().commandLine(), "<" + base + "> advertises no lds:admin link: --file needs the end-user base URI"); + List ontologies = getLinkTargets(base, LDS.ontology.getURI()); + if (ontologies.isEmpty()) throw new IllegalStateException("<" + base + "> advertises no lds:ontology link"); + URI ontology = ontologies.get(0); + URI doc = graph != null ? graph : adminBase.resolve(NAMESPACE_DOCUMENT_PATH); + + // parsed against the ontology IRI, so that <#> in the file is the ontology itself (RFC 3986 drops the base's fragment) + Model file = path != null ? readModel(mediaType, ontology, path) : readModel(mediaType, ontology, System.in); + + Model current; + try (Response response = HttpException.check(doc, getClient().get(doc, ACCEPT_TURTLE))) + { + current = response.readEntity(Model.class); + } + + put(getClient(), doc, file.add(header(current, current.createResource(doc.toString())))); + clear(adminBase, ontology); + + print(doc); + + return 0; + } + + /** + * Returns the statements of a document that a re-install keeps: those about the document itself and about its + * primary topics, the ontology header the platform seeds. The same statements patch-ontology.ru kept. + * + * @param model document model + * @param doc document resource + * @return header statements + */ + public static Model header(Model model, Resource doc) + { + Model header = ModelFactory.createDefaultModel(); + header.add(model.listStatements(doc, null, (RDFNode)null)); + model.listObjectsOfProperty(doc, FOAF.primaryTopic).filterKeep(RDFNode::isResource). + forEach(topic -> header.add(model.listStatements(topic.asResource(), null, (RDFNode)null))); + + return header; + } + + /** + * Clears an ontology from the admin application's cache, reloading it. + * + * @param adminBase admin application base URI + * @param ontology ontology URI + */ + protected void clear(URI adminBase, URI ontology) + { + URI target = URI.create(adminBase + "clear"); + + HttpException.check(target, getClient().postForm(target, new Form("uri", ontology.toString()), ACCEPT_TURTLE)).close(); + } + /** * Fetches the source ontology through the Linked Data proxy, which converts any Jena-parseable * format to RDF/XML. diff --git a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/imports/ImportCSV.java b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/imports/ImportCSV.java index 5b29f0ead..adf29dd26 100644 --- a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/imports/ImportCSV.java +++ b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/imports/ImportCSV.java @@ -24,6 +24,7 @@ import java.nio.file.Files; import java.nio.file.Path; import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; @@ -31,6 +32,9 @@ * Imports CSV data: adds the transformation query, uploads the CSV file and creates the * import metadata on the target document. Mirrors bin/imports/import-csv.sh, * calling the same steps in-process instead of via subscripts. + *

+ * Prints the URI of the import it creates, a fragment of the target document, as the only stdout line. The + * import runs in the background on the server; --wait waits for it to end ({@link ImportWait}). * * @author Martynas Jusevičius {@literal } */ @@ -53,9 +57,12 @@ public class ImportCSV extends BaseCommand @Option(names = "--delimiter", defaultValue = ",", paramLabel = "CHAR", description = "CSV delimiter character (default: ${DEFAULT-VALUE})") private String delimiter; - @Parameters(paramLabel = "TARGET_URI", description = "URI of the import document") + @Parameters(paramLabel = "TARGET_URI", description = "URI of the document the import is added to") private URI target; + @Mixin + private ImportWait importWait; + @Override public Integer call() throws Exception { @@ -65,10 +72,12 @@ public Integer call() throws Exception URI fileURI = AddFile.core(getClient(), target, csvFile, "text/csv", title, null); - AddCSVImport.core(getClient(), target, null, title, query, fileURI, delimiter, description); - print(target); + String importId = Slugs.defaultSlug(); + AddCSVImport.core(getClient(), target, "#" + importId, title, query, fileURI, delimiter, description); + URI importURI = target.resolve("#" + importId); + print(importURI); - return 0; + return importWait.isWait() ? importWait.await(getClient(), importURI, getSpec().commandLine().getErr()) : 0; } } diff --git a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/imports/ImportRDF.java b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/imports/ImportRDF.java index 08a87853b..162ceaf15 100644 --- a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/imports/ImportRDF.java +++ b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/imports/ImportRDF.java @@ -24,6 +24,7 @@ import java.nio.file.Files; import java.nio.file.Path; import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; @@ -31,6 +32,9 @@ * Imports RDF data: optionally adds a transformation query, uploads the RDF file and creates * the import metadata on the target document. Mirrors bin/imports/import-rdf.sh, * calling the same steps in-process instead of via subscripts. + *

+ * Prints the URI of the import it creates, a fragment of the target document, as the only stdout line. The + * import runs in the background on the server; --wait waits for it to end ({@link ImportWait}). * * @author Martynas Jusevičius {@literal } */ @@ -56,9 +60,12 @@ public class ImportRDF extends BaseCommand @Option(names = "--graph", paramLabel = "GRAPH_URI", description = "URI of the target named graph (optional)") private URI graph; - @Parameters(paramLabel = "TARGET_URI", description = "URI of the import document") + @Parameters(paramLabel = "TARGET_URI", description = "URI of the document the import is added to") private URI target; + @Mixin + private ImportWait importWait; + @Override public Integer call() throws Exception { @@ -74,9 +81,10 @@ public Integer call() throws Exception String importId = Slugs.defaultSlug(); AddRDFImport.core(getClient(), target, "#" + importId, title, fileURI, query, query == null ? graph : null, description); - print(target); + URI importURI = target.resolve("#" + importId); + print(importURI); - return 0; + return importWait.isWait() ? importWait.await(getClient(), importURI, getSpec().commandLine().getErr()) : 0; } } diff --git a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/imports/ImportWait.java b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/imports/ImportWait.java new file mode 100644 index 000000000..7559b3ad9 --- /dev/null +++ b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/command/imports/ImportWait.java @@ -0,0 +1,137 @@ +/* + * Copyright 2026 Martynas Jusevičius . + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.atomgraph.linkeddatahub.cli.command.imports; + +import com.atomgraph.linkeddatahub.cli.http.HttpException; +import com.atomgraph.linkeddatahub.cli.http.LDHClient; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import java.io.PrintWriter; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.Property; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.ResourceFactory; +import org.apache.jena.vocabulary.DCTerms; +import picocli.CommandLine.Option; + +/** + * The --wait option of the import commands: an import runs in the background on the server, and this + * waits for it to end and reports how it ended. + *

+ * The platform adds prov:endedAtTime to the import when it finishes, and its outcome is an entity that + * points back at it with prov:wasGeneratedBy: a void:Dataset with a triple count on success, + * one with a dct:description message on failure. + * + * @author Martynas Jusevičius {@literal } + */ +public class ImportWait +{ + + private static final MediaType[] ACCEPT_NTRIPLES = { com.atomgraph.core.MediaType.APPLICATION_NTRIPLES_TYPE }; + private static final String PROV = "http://www.w3.org/ns/prov#"; + private static final Property ENDED_AT_TIME = ResourceFactory.createProperty(PROV + "endedAtTime"); + private static final Property WAS_GENERATED_BY = ResourceFactory.createProperty(PROV + "wasGeneratedBy"); + private static final Property TRIPLES = ResourceFactory.createProperty("http://rdfs.org/ns/void#triples"); + /** Interval between two reads of the import document, in milliseconds */ + private static final long POLL_INTERVAL = 5000; + + @Option(names = "--wait", description = "Wait for the import to end; exits 1 if it failed or timed out") + private boolean wait; + + @Option(names = "--timeout", defaultValue = "1200", paramLabel = "SECONDS", description = "How long --wait waits (default: ${DEFAULT-VALUE})") + private long timeout; + + /** + * Returns true if the import should be waited for. + * + * @return true with --wait + */ + public boolean isWait() + { + return wait; + } + + /** + * Waits for an import to end, polling its document. + * + * @param client HTTP client + * @param importURI URI of the import resource + * @param err stream for progress and errors + * @return exit code: 0 if the import ended without errors, 1 if it failed or did not end in time + * @throws InterruptedException if the wait is interrupted + * @throws java.net.URISyntaxException if the import URI is malformed + */ + public int await(LDHClient client, URI importURI, PrintWriter err) throws InterruptedException, java.net.URISyntaxException + { + URI doc = new URI(importURI.getScheme(), importURI.getSchemeSpecificPart(), null); // the document the import is in + long deadline = System.currentTimeMillis() + timeout * 1000; + + while (true) + { + Model model; + try (Response response = HttpException.check(doc, client.get(doc, ACCEPT_NTRIPLES))) + { + model = response.readEntity(Model.class); + } + + Resource imp = model.createResource(importURI.toString()); + if (imp.hasProperty(ENDED_AT_TIME)) return outcome(model, imp, err); + + if (System.currentTimeMillis() >= deadline) + { + err.println("Timed out waiting for <" + importURI + ">"); + return 1; + } + + Thread.sleep(POLL_INTERVAL); + } + } + + /** + * Reports the outcome of an ended import. + * + * @param model import document model + * @param imp import resource + * @param err stream for the report + * @return exit code: 0 without errors, 1 with any + */ + protected static int outcome(Model model, Resource imp, PrintWriter err) + { + List errors = new ArrayList<>(); + long triples = 0; + + for (Resource entity : model.listSubjectsWithProperty(WAS_GENERATED_BY, imp).toList()) + { + if (entity.hasProperty(DCTerms.description)) errors.add(entity.getProperty(DCTerms.description).getString()); + if (entity.hasProperty(TRIPLES)) triples += entity.getProperty(TRIPLES).getLong(); + } + + if (!errors.isEmpty()) + { + err.println("Import <" + imp + "> failed:"); + errors.forEach(err::println); + return 1; + } + + err.println("Import <" + imp + "> done: " + triples + " triples"); + return 0; + } + +} diff --git a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/http/LDHClient.java b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/http/LDHClient.java index af71a62fb..e5f9556ff 100644 --- a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/http/LDHClient.java +++ b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/http/LDHClient.java @@ -18,10 +18,8 @@ import com.atomgraph.core.MediaTypes; import com.atomgraph.core.client.GraphStoreClient; -import com.atomgraph.linkeddatahub.cli.util.URIRewriter; import jakarta.ws.rs.client.Client; import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.client.WebTarget; import jakarta.ws.rs.core.EntityTag; import jakarta.ws.rs.core.Form; import jakarta.ws.rs.core.HttpHeaders; @@ -35,8 +33,8 @@ /** * Graph Store Protocol client for LinkedDataHub documents (direct graph identification), * extended with SPARQL update over PATCH and form-encoded POST. When a proxy URI is given, - * every request URI has its origin rewritten to the proxy's origin, matching the - * --proxy handling of the bin/ shell scripts. + * every request connects to the proxy and keeps its logical Host + * ({@link ProxyRewriteFilter}), so dataspaces stay routed by host and responses carry logical URIs. * * @author Martynas Jusevičius {@literal } */ @@ -53,20 +51,14 @@ public class LDHClient extends GraphStoreClient * * @param client Jersey client with WebID client certificate * @param mediaTypes registry of readable/writable media types - * @param proxy proxy URI whose origin replaces the request URI origin (optional, can be null) + * @param proxy proxy URI that receives the connections (optional, can be null) */ public LDHClient(Client client, MediaTypes mediaTypes, URI proxy) { - super(client, mediaTypes); + super(proxy != null ? client.register(new ProxyRewriteFilter(proxy)) : client, mediaTypes); this.proxy = proxy; } - @Override - protected WebTarget getWebTarget(URI uri) - { - return super.getWebTarget(getProxy() != null ? URIRewriter.rewrite(uri, getProxy()) : uri); - } - /** * Patches a document with a SPARQL update. * diff --git a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/http/Links.java b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/http/Links.java new file mode 100644 index 000000000..60cecb755 --- /dev/null +++ b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/http/Links.java @@ -0,0 +1,85 @@ +/* + * Copyright 2026 Martynas Jusevičius . + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.atomgraph.linkeddatahub.cli.http; + +import jakarta.ws.rs.core.HttpHeaders; +import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Reads Link response headers (RFC 8288). + *

+ * The platform folds all its links into one comma-separated header, so entries are told apart by their + * <target>, not by splitting on commas, which a target URI may contain. + * + * @author Martynas Jusevičius {@literal } + */ +public final class Links +{ + + private static final Pattern ENTRY = Pattern.compile("<([^>]+)>([^<]*)"); + private static final Pattern REL = Pattern.compile("(?:^|;)\\s*rel\\s*=\\s*(?:\"([^\"]*)\"|([^\\s;,]+))"); + + private Links() { } + + /** + * Returns the targets of the response's links with the given relation. + * + * @param response HTTP response + * @param rel relation URI or name + * @return link targets, in header order + */ + public static List targets(Response response, String rel) + { + List targets = new ArrayList<>(); + + for (String header : response.getStringHeaders().getOrDefault(HttpHeaders.LINK, List.of())) + targets.addAll(targets(header, rel)); + + return targets; + } + + /** + * Returns the targets of the links in a header value with the given relation. + * + * @param header Link header value, possibly with several comma-separated links + * @param rel relation URI or name + * @return link targets, in header order + */ + public static List targets(String header, String rel) + { + List targets = new ArrayList<>(); + Matcher entry = ENTRY.matcher(header); + + while (entry.find()) + { + Matcher params = REL.matcher(entry.group(2)); + if (!params.find()) continue; + + String rels = params.group(1) != null ? params.group(1) : params.group(2); + if (Arrays.asList(rels.trim().split("\\s+")).contains(rel)) targets.add(URI.create(entry.group(1))); + } + + return targets; + } + +} diff --git a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/http/ProxyRewriteFilter.java b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/http/ProxyRewriteFilter.java new file mode 100644 index 000000000..7f7a1c174 --- /dev/null +++ b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/http/ProxyRewriteFilter.java @@ -0,0 +1,71 @@ +/* + * Copyright 2026 Martynas Jusevičius . + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.atomgraph.linkeddatahub.cli.http; + +import jakarta.ws.rs.client.ClientRequestContext; +import jakarta.ws.rs.client.ClientRequestFilter; +import jakarta.ws.rs.core.HttpHeaders; +import jakarta.ws.rs.core.UriBuilder; +import java.net.URI; + +/** + * Sends every request to the proxy while keeping the logical Host. + *

+ * The connection goes to the proxy's scheme, host and port; the path, query and the Host header stay those + * of the logical URI. The platform routes dataspaces by Host, so an admin request through the end-user + * proxy still reaches the admin dataspace, and the URIs the platform answers with are the logical ones. + * + * @author Martynas Jusevičius {@literal } + */ +public class ProxyRewriteFilter implements ClientRequestFilter +{ + + private final URI proxy; + + /** + * Constructs the filter. + * + * @param proxy proxy URI whose scheme, host and port receive the connection (any path is ignored) + */ + public ProxyRewriteFilter(URI proxy) + { + if (proxy == null || proxy.getScheme() == null || proxy.getHost() == null) throw new IllegalArgumentException("Proxy URI '" + proxy + "' is not absolute"); + this.proxy = proxy; + } + + @Override + public void filter(ClientRequestContext request) + { + URI uri = request.getUri(); + if (uri.getRawAuthority() == null) throw new IllegalArgumentException("URI '" + uri + "' is not absolute"); + + request.getHeaders().putSingle(HttpHeaders.HOST, uri.getRawAuthority()); + // UriBuilder rather than string surgery, so an encoded path or query survives untouched + request.setUri(UriBuilder.fromUri(uri).scheme(getProxy().getScheme()).host(getProxy().getHost()).port(getProxy().getPort()).build()); + } + + /** + * Returns the proxy URI. + * + * @return proxy URI + */ + public URI getProxy() + { + return proxy; + } + +} diff --git a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/util/URIRewriter.java b/cli/src/main/java/com/atomgraph/linkeddatahub/cli/util/URIRewriter.java deleted file mode 100644 index 2bb2a603d..000000000 --- a/cli/src/main/java/com/atomgraph/linkeddatahub/cli/util/URIRewriter.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2026 Martynas Jusevičius . - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.atomgraph.linkeddatahub.cli.util; - -import java.net.URI; - -/** - * Sends a request somewhere other than where its URI says. - *

- * This is the --proxy option: a document's URI identifies it, but the host that - * answers for it may be a different one - the client-certificate port, say, or a tunnel. The - * logical URI stays in the request; only the origin it is sent to changes. - *

- * A CLI concern, not a document-shape one, which is why it stays here while the URI conventions - * the platform itself applies live in {@link com.atomgraph.linkeddatahub.rdf.URIs}. - * - * @author Martynas Jusevičius {@literal } - */ -public final class URIRewriter -{ - - private URIRewriter() { } - - /** - * Replaces the origin (scheme and authority) of a URI with the origin of the proxy URI, - * keeping the path, query and fragment. - * - * @param uri logical URI - * @param proxy proxy URI whose origin is substituted - * @return rewritten URI - */ - public static URI rewrite(URI uri, URI proxy) - { - return URI.create(origin(proxy) + uri.toString().substring(origin(uri).length())); - } - - /** - * Returns the origin (scheme and authority) of a URI. - * - * @param uri URI - * @return origin string, e.g. https://localhost:4443 - */ - private static String origin(URI uri) - { - if (uri.getScheme() == null || uri.getRawAuthority() == null) throw new IllegalArgumentException("URI '" + uri + "' is not absolute"); - - return uri.getScheme() + "://" + uri.getRawAuthority(); - } - -} diff --git a/cli/src/test/java/com/atomgraph/linkeddatahub/cli/AdminDiscoveryTest.java b/cli/src/test/java/com/atomgraph/linkeddatahub/cli/AdminDiscoveryTest.java new file mode 100644 index 000000000..a6d6c77fe --- /dev/null +++ b/cli/src/test/java/com/atomgraph/linkeddatahub/cli/AdminDiscoveryTest.java @@ -0,0 +1,198 @@ +/* + * Copyright 2026 Martynas Jusevičius . + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.atomgraph.linkeddatahub.cli; + +import static com.atomgraph.linkeddatahub.cli.CommandOutputTest.commandLine; +import static com.atomgraph.linkeddatahub.cli.CommandOutputTest.keyStorePath; +import com.atomgraph.linkeddatahub.cli.http.StubServer; +import java.io.StringReader; +import java.io.StringWriter; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFParser; +import org.apache.jena.vocabulary.OWL; +import org.apache.jena.vocabulary.RDF; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests that admin commands find the admin application from the end-user base's lds:admin link, that a + * proxy keeps the logical Host, and that import ontology --file replaces the namespace + * ontology document keeping its header. + * + * @author Martynas Jusevičius {@literal } + */ +public class AdminDiscoveryTest +{ + + private static final String LDS = "https://w3id.org/atomgraph/linkeddatahub/dataspaces#"; + + private static String link(URI target, String rel) + { + return "<" + target + ">; rel=\"" + rel + "\""; + } + + private static List methodsAndTargets(StubServer server) + { + return server.getRequests().stream().map(r -> r.method() + " " + r.target()).collect(Collectors.toList()); + } + + @Test + public void clearFollowsAdminLinkFromEndUserBase() throws Exception + { + try (StubServer endUser = new StubServer(); StubServer admin = new StubServer()) + { + endUser.respondsWithHeader("Link", link(admin.baseURI(), LDS + "admin")); + admin.responds(200, ""); + StringWriter out = new StringWriter(), err = new StringWriter(); + + int code = commandLine(out, err).execute("admin", "clear", "ontology", "-c", keyStorePath().toString(), "-p", "changeit", + endUser.baseURI().toString()); + + assertEquals(0, code, err.toString()); + assertEquals("HEAD", endUser.getLastMethod()); + assertEquals("POST", admin.getLastMethod()); + assertEquals("/clear", admin.getLastTarget()); + } + } + + @Test + public void baseWithoutAdminLinkIsTakenAsAdminBase() throws Exception + { + try (StubServer admin = new StubServer()) + { + admin.responds(200, ""); + StringWriter out = new StringWriter(), err = new StringWriter(); + + int code = commandLine(out, err).execute("admin", "clear", "ontology", "-c", keyStorePath().toString(), "-p", "changeit", + admin.baseURI().toString()); + + assertEquals(0, code, err.toString()); + assertEquals(List.of("HEAD /", "POST /clear"), methodsAndTargets(admin)); + } + } + + @Test + public void proxyKeepsLogicalHostAndPath() throws Exception + { + try (StubServer proxy = new StubServer()) + { + proxy.respondsWithType("application/n-triples").responds(200, " ."); + StringWriter out = new StringWriter(), err = new StringWriter(); + + int code = commandLine(out, err).execute("get", "-c", keyStorePath().toString(), "-p", "changeit", + "--accept", "application/n-triples", "--proxy", proxy.baseURI().toString(), "http://admin.example.org:4443/a%20b/?c=d"); + + assertEquals(0, code, err.toString()); + assertEquals("/a%20b/?c=d", proxy.getLastTarget()); + assertEquals("admin.example.org:4443", proxy.getLastHeader("Host")); + } + } + + @Test + public void importOntologyFileReplacesDocumentKeepingHeader(@TempDir Path dir) throws Exception + { + try (StubServer endUser = new StubServer(); StubServer admin = new StubServer()) + { + URI ontology = endUser.baseURI().resolve("ns#"); + URI doc = admin.baseURI().resolve("ontologies/namespace/"); + endUser.respondsWithHeader("Link", link(admin.baseURI(), LDS + "admin") + ", " + link(ontology, LDS + "ontology")); + // the seeded document, with a term from a previous install that the file no longer has + admin.respondsTo("/ontologies/namespace/", 200, "text/turtle", + "<" + doc + "> a ; " + + " \"Namespace\" ; <" + ontology + "> .\n" + + "<" + ontology + "> a ; .\n" + + "<" + ontology.resolve("#Removed") + "> a .\n"); + admin.responds(200, ""); + + Path file = dir.resolve("ns.ttl"); + Files.writeString(file, "@prefix : <#> . @prefix owl: . : a owl:Ontology . :Book a owl:Class ."); + StringWriter out = new StringWriter(), err = new StringWriter(); + + int code = commandLine(out, err).execute("admin", "import", "ontology", "-c", keyStorePath().toString(), "-p", "changeit", + "--file", file.toString(), endUser.baseURI().toString()); + + assertEquals(0, code, err.toString()); + assertEquals(doc.toString(), out.toString().strip()); + + StubServer.Request put = admin.getRequests().stream().filter(r -> r.method().equals("PUT")).findFirst().orElseThrow(); + Model written = ModelFactory.createDefaultModel(); + RDFParser.create().source(new StringReader(put.body())).lang(Lang.TURTLE).base(doc.toString()).parse(written); + + assertTrue(written.contains(written.createResource(ontology.resolve("#Book").toString()), RDF.type, OWL.Class), "the file's term"); + assertTrue(written.contains(written.createResource(ontology.toString()), OWL.imports, written.createResource("https://w3id.org/atomgraph/linkeddatahub/default#")), "the seeded header"); + assertTrue(written.contains(written.createResource(doc.toString()), null, (org.apache.jena.rdf.model.RDFNode)null), "the document's own description"); + assertFalse(written.contains(written.createResource(ontology.resolve("#Removed").toString()), null, (org.apache.jena.rdf.model.RDFNode)null), "a term the file dropped"); + + StubServer.Request clear = admin.getRequests().stream().filter(r -> r.method().equals("POST")).findFirst().orElseThrow(); + assertEquals("/clear", clear.target()); + assertTrue(clear.body().contains("uri=" + java.net.URLEncoder.encode(ontology.toString(), java.nio.charset.StandardCharsets.UTF_8)), clear.body()); + } + } + + @Test + public void importOntologyFileNeedsEndUserBase(@TempDir Path dir) throws Exception + { + try (StubServer admin = new StubServer()) + { + admin.responds(200, ""); + Path file = dir.resolve("ns.ttl"); + Files.writeString(file, "<#> a ."); + StringWriter out = new StringWriter(), err = new StringWriter(); + + int code = commandLine(out, err).execute("admin", "import", "ontology", "-c", keyStorePath().toString(), "-p", "changeit", + "--file", file.toString(), admin.baseURI().toString()); + + assertEquals(2, code); + assertTrue(err.toString().contains("end-user base"), err.toString()); + } + } + + @Test + public void importCsvPrintsTheImportURI(@TempDir Path dir) throws Exception + { + try (StubServer server = new StubServer()) + { + server.responds(200, ""); + Path query = Files.writeString(dir.resolve("q.rq"), "CONSTRUCT WHERE { ?s ?p ?o }"); + Path csv = Files.writeString(dir.resolve("d.csv"), "a,b\n1,2\n"); + URI target = server.baseURI().resolve("items/"); + StringWriter out = new StringWriter(), err = new StringWriter(); + + int code = commandLine(out, err).execute("import", "csv", "-c", keyStorePath().toString(), "-p", "changeit", + "--title", "Items", "--query-file", query.toString(), "--csv-file", csv.toString(), target.toString()); + + assertEquals(0, code, err.toString()); + String printed = out.toString().strip(); + assertTrue(printed.matches(java.util.regex.Pattern.quote(target.toString()) + "#[0-9a-f-]{36}"), printed); + // the printed import is the resource the import request describes + String importPost = server.getRequests().stream().filter(r -> r.method().equals("POST") && r.body().contains("CSVImport")). + map(StubServer.Request::body).findFirst().orElseThrow(); + assertTrue(importPost.contains(printed.substring(printed.indexOf('#'))), importPost); + } + } + +} diff --git a/cli/src/test/java/com/atomgraph/linkeddatahub/cli/CommandParsingTest.java b/cli/src/test/java/com/atomgraph/linkeddatahub/cli/CommandParsingTest.java index 13d5f9bab..2b1ea5136 100644 --- a/cli/src/test/java/com/atomgraph/linkeddatahub/cli/CommandParsingTest.java +++ b/cli/src/test/java/com/atomgraph/linkeddatahub/cli/CommandParsingTest.java @@ -99,6 +99,15 @@ public void missingRequiredOptionIsUsageError() assertEquals(CommandLine.ExitCode.USAGE, commandLine().execute("create", "item", "--container", "https://localhost:4443/some/")); } + /** import ontology imports one thing: an external ontology or a local file, never both or neither. */ + @Test + public void importOntologyTakesSourceOrFile() + { + assertEquals(CommandLine.ExitCode.USAGE, commandLine().execute("admin", "import", "ontology", "https://localhost:4443/")); + assertEquals(CommandLine.ExitCode.USAGE, commandLine().execute("admin", "import", "ontology", + "--source", "https://example.org/ns#", "--file", "ns.ttl", "https://localhost:4443/")); + } + @Test public void unknownOptionIsUsageError() { diff --git a/cli/src/test/java/com/atomgraph/linkeddatahub/cli/command/imports/ImportWaitTest.java b/cli/src/test/java/com/atomgraph/linkeddatahub/cli/command/imports/ImportWaitTest.java new file mode 100644 index 000000000..f4c939bd3 --- /dev/null +++ b/cli/src/test/java/com/atomgraph/linkeddatahub/cli/command/imports/ImportWaitTest.java @@ -0,0 +1,76 @@ +/* + * Copyright 2026 Martynas Jusevičius . + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.atomgraph.linkeddatahub.cli.command.imports; + +import java.io.PrintWriter; +import java.io.StringReader; +import java.io.StringWriter; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFParser; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.api.Test; + +/** + * Tests how an ended import's outcome is reported, in the shape the platform writes it. + * + * @author Martynas Jusevičius {@literal } + */ +public class ImportWaitTest +{ + + private static final String IMPORT = "https://localhost:4443/authors/#id837da0c2"; + + private static Model model(String ntriples) + { + Model model = ModelFactory.createDefaultModel(); + RDFParser.create().source(new StringReader(ntriples)).lang(Lang.NTRIPLES).parse(model); + return model; + } + + @Test + public void successReportsTripleCount() + { + Model model = model(""" + "2026-10-16T10:50:47.666Z"^^ . + . + "16"^^ . + . + """); + StringWriter err = new StringWriter(); + + assertEquals(0, ImportWait.outcome(model, model.createResource(IMPORT), new PrintWriter(err, true))); + assertTrue(err.toString().contains("16 triples"), err.toString()); + } + + @Test + public void failureReportsMessageAndExitsOne() + { + Model model = model(""" + "2026-10-16T10:50:47.666Z"^^ . + . + "Missing dct:title" . + """); + StringWriter err = new StringWriter(); + + assertEquals(1, ImportWait.outcome(model, model.createResource(IMPORT), new PrintWriter(err, true))); + assertTrue(err.toString().contains("Missing dct:title"), err.toString()); + } + +} diff --git a/cli/src/test/java/com/atomgraph/linkeddatahub/cli/http/LinksTest.java b/cli/src/test/java/com/atomgraph/linkeddatahub/cli/http/LinksTest.java new file mode 100644 index 000000000..219591617 --- /dev/null +++ b/cli/src/test/java/com/atomgraph/linkeddatahub/cli/http/LinksTest.java @@ -0,0 +1,73 @@ +/* + * Copyright 2026 Martynas Jusevičius . + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.atomgraph.linkeddatahub.cli.http; + +import java.net.URI; +import java.util.List; +import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; + +/** + * Tests reading link targets from the folded Link header the platform sends. + * + * @author Martynas Jusevičius {@literal } + */ +public class LinksTest +{ + + private static final String ADMIN = "https://w3id.org/atomgraph/linkeddatahub/dataspaces#admin"; + private static final String ONTOLOGY = "https://w3id.org/atomgraph/linkeddatahub/dataspaces#ontology"; + /** The shape ResponseHeadersFilter folds its links into: one header, comma-separated */ + private static final String FOLDED = "; rel=\"http://www.w3.org/ns/sparql-service-description#endpoint\", " + + "; rel=\"https://w3id.org/atomgraph/linkeddatahub/dataspaces#dataspace\", " + + "; rel=\"" + ONTOLOGY + "\", " + + "; rel=\"" + ADMIN + "\""; + + @Test + public void findsTargetByRelInFoldedHeader() + { + assertEquals(List.of(URI.create("https://admin.localhost:4443/")), Links.targets(FOLDED, ADMIN)); + assertEquals(List.of(URI.create("https://localhost:4443/ns#")), Links.targets(FOLDED, ONTOLOGY)); + } + + @Test + public void absentRelYieldsNothing() + { + assertEquals(List.of(), Links.targets(FOLDED, "timemap")); + } + + @Test + public void relMustMatchWholeNotPrefix() + { + // lds:dataspace must not match a query for a rel that is its prefix + assertEquals(List.of(), Links.targets(FOLDED, "https://w3id.org/atomgraph/linkeddatahub/dataspaces#data")); + } + + @Test + public void targetWithCommaIsNotSplit() + { + assertEquals(List.of(URI.create("https://localhost:4443/a,b/")), Links.targets("; rel=\"timemap\"", "timemap")); + } + + @Test + public void unquotedAndMultipleRels() + { + assertEquals(List.of(URI.create("https://localhost:4443/tm")), Links.targets("; rel=timemap", "timemap")); + assertEquals(List.of(URI.create("https://localhost:4443/x")), Links.targets("; rel=\"self timemap\"", "timemap")); + } + +} diff --git a/cli/src/test/java/com/atomgraph/linkeddatahub/cli/util/URIRewriterTest.java b/cli/src/test/java/com/atomgraph/linkeddatahub/cli/util/URIRewriterTest.java deleted file mode 100644 index 276d4b536..000000000 --- a/cli/src/test/java/com/atomgraph/linkeddatahub/cli/util/URIRewriterTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2026 Martynas Jusevičius . - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.atomgraph.linkeddatahub.cli.util; - -import java.net.URI; -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -/** - * Tests for {@link URIRewriter}. - */ -public class URIRewriterTest -{ - - @Test - public void rewriteReplacesOriginKeepingPathQueryFragment() - { - assertEquals(URI.create("https://localhost:8443/a%20b/c/?d=e#f"), - URIRewriter.rewrite(URI.create("https://linkeddatahub.com/a%20b/c/?d=e#f"), URI.create("https://localhost:8443"))); - } - - @Test - public void rewriteIgnoresProxyPath() - { - assertEquals(URI.create("https://localhost:8443/some/"), - URIRewriter.rewrite(URI.create("https://linkeddatahub.com:4443/some/"), URI.create("https://localhost:8443/ignored/"))); - } - - @Test - public void rewriteRejectsRelativeURI() - { - assertThrows(IllegalArgumentException.class, - () -> URIRewriter.rewrite(URI.create("/relative/path"), URI.create("https://localhost:8443"))); - } - -} diff --git a/docker-compose.yml b/docker-compose.yml index ff9d76b2e..ee1160d5a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,8 @@ secrets: file: ./secrets/secretary_cert_password.txt client_truststore_password: file: ./secrets/client_truststore_password.txt + # openai_api_key: + # file: ./secrets/openai_api_key.txt # google_client_id: # file: ./secrets/google/client_id.txt # google_client_secret: @@ -28,6 +30,8 @@ services: depends_on: linkeddatahub: condition: service_healthy + web-algebra: + condition: service_started # its upstream has to resolve when nginx starts ports: - ${HTTP_PORT}:8080 # allow Tomcat to do HTTP to HTTPS redirect - ${HTTPS_PORT}:8443 # HTTPS @@ -132,8 +136,10 @@ services: # every outbound request (SPARQL SERVICE, LOAD) goes through egress, which only lets it reach public addresses. # JAVA_TOOL_OPTIONS rather than JAVA_OPTIONS, which an override setting the heap would replace. The empty # nonProxyHosts is load-bearing: by default the JVM bypasses the proxy for localhost, and - # SERVICE would read a sibling dataset in this same server - - JAVA_TOOL_OPTIONS=-Dhttp.proxyHost=${EGRESS_PROXY_HOST:-egress} -Dhttp.proxyPort=3128 -Dhttps.proxyHost=${EGRESS_PROXY_HOST:-egress} -Dhttps.proxyPort=3128 -Dhttp.nonProxyHosts= + # SERVICE would read a sibling dataset in this same server. + # ExitOnOutOfMemoryError: a query that exhausts the heap kills the JVM and restart: on-failure brings it back in + # seconds - the query timeout bounds time, not memory, and a JVM past an OutOfMemoryError keeps running broken + - JAVA_TOOL_OPTIONS=-Dhttp.proxyHost=${EGRESS_PROXY_HOST:-egress} -Dhttp.proxyPort=3128 -Dhttps.proxyHost=${EGRESS_PROXY_HOST:-egress} -Dhttps.proxyPort=3128 -Dhttp.nonProxyHosts= -XX:+ExitOnOutOfMemoryError expose: - 3030 volumes: @@ -149,6 +155,38 @@ services: target: /etc/squid/squid.conf expose: - 3128 + web-algebra: # runs Web-Algebra plans against this instance, and writes them from natural language for the assistant drawer + image: ghcr.io/atomgraph/webalgebra-server-ldh:${WEBALGEBRA_VERSION:-latest} + mem_limit: 1024m + restart: on-failure + depends_on: + - egress + expose: + - 8080 # internal only: nginx is the sole client, and the server trusts the Client-Cert header only because nginx sets it + environment: + # the origin that receives an identity - this instance, with every dataspace subdomain under it. Its public name is + # not a route to it from inside the network, so requests for it go to nginx's client-cert listener under that name + - WEBALGEBRA_TRUSTED_ORIGIN=${PROTOCOL}://${HOST}:${HTTPS_PORT} + - WEBALGEBRA_PROXY_HOST=nginx + - WEBALGEBRA_PROXY_PORT=9443 + # the secretary's WebID: the agent every user's WebID document already delegates (root-owner.trig.template, SignUp), + # so a plan submitted by a user runs under that user's own access control, not the secretary's + - WEBALGEBRA_KEYSTORE=/var/webalgebra/ssl/secretary/keystore.p12 + - WEBALGEBRA_KEYSTORE_PASSWORD_FILE=/run/secrets/secretary_cert_password + - WEBALGEBRA_TRUSTSTORE=/var/webalgebra/ssl/server/server.crt # the self-signed server certificate, as PEM + # the model key is optional, like the OAuth secrets: a deployment that writes plans declares the openai_api_key + # secret and this variable in its override; without them the service runs plans and declines to write any + # - OPENAI_API_KEY_FILE=/run/secrets/openai_api_key + - OPENAI_MODEL=${OPENAI_MODEL:-gpt-4o} + # public destinations (SPARQL endpoints a plan names, the model's API) go through egress like the platform's own; nginx + # is reached directly, since egress refuses internal addresses + - JAVA_TOOL_OPTIONS=-Dhttp.proxyHost=${EGRESS_PROXY_HOST:-egress} -Dhttp.proxyPort=3128 -Dhttps.proxyHost=${EGRESS_PROXY_HOST:-egress} -Dhttps.proxyPort=3128 -Dhttp.nonProxyHosts=nginx + secrets: + - secretary_cert_password + # - openai_api_key + volumes: + - ./ssl/secretary:/var/webalgebra/ssl/secretary:ro + - ./ssl/server:/var/webalgebra/ssl/server:ro varnish-frontend: image: varnish:7.7.3 user: root # otherwise varnish user does not have permissions to the mounted folder which is owner by root @@ -200,6 +238,10 @@ configs: } http { + upstream web-algebra { + server web-algebra:8080; + } + upstream linkeddatahub { server ${NGINX_UPSTREAM_SERVER:-varnish-frontend:6060}; } @@ -217,8 +259,26 @@ configs: limit_req_zone $$limit_key zone=linked_data:10m rate=15r/s; limit_req_zone $$limit_key zone=static_files:10m rate=20r/s; + limit_req_zone $$limit_key zone=web_algebra:10m rate=5r/s; # a plan is a model call; polling is what bursts limit_req_status 429; + # SPARQL endpoints are capped by concurrency, not only by rate: a handful of heavy queries in flight at once is + # what exhausts the store's heap, however slowly they arrive. The keys are set for an external client's + # /sparql request only - every other path and every internal (RFC 1918) caller maps to "", which nginx does not + # count - so the caps sit in the generic locations below without touching anything else + map $$uri $$sparql_key { + default ""; + /sparql $$limit_key; + } + map $$sparql_key $$sparql_all { + default sparql; + "" ""; + } + limit_conn_zone $$sparql_key zone=sparql_per_client:10m; + limit_conn_zone $$sparql_all zone=sparql_total:1m; + limit_req_zone $$sparql_key zone=sparql:10m rate=${NGINX_SPARQL_RATE:-2r/s}; + limit_conn_status 429; + client_max_body_size ${MAX_CONTENT_LENGTH:-2097152}; # server with optional client cert authentication @@ -231,6 +291,30 @@ configs: ssl_prefer_server_ciphers on; ssl_verify_client ${NGINX_SSL_VERIFY_CLIENT:-optional_no_ca}; + # the assistant's plan service. A reserved path, like /static/ and /uploads/: prefix-matched ahead of the + # generic location and never rewritten, so /webalgebra, /webalgebra/plans and /webalgebra/{id}/result reach the + # sidecar as they are. Reachable only with a WebID certificate on the connection: the sidecar acts for the + # WebID in the certificate nginx forwards, and a caller without one has nobody to act for. On-Behalf-Of is + # cleared for the same reason Client-Cert is - identity comes from the connection, never from a header + location ^~ /webalgebra { + if ($$ssl_client_verify = NONE) { + return 401; + } + + proxy_pass http://web-algebra; + limit_req zone=web_algebra burst=10 nodelay; + proxy_read_timeout 120s; # writing a plan is a 10-30 s model call + + proxy_set_header Host $$host; + proxy_set_header X-Forwarded-Host $$host; + proxy_set_header X-Forwarded-Proto $$scheme; + proxy_set_header X-Forwarded-Port ${HTTPS_PORT}; + + proxy_set_header Client-Cert ''; + proxy_set_header Client-Cert $$ssl_client_escaped_cert; + proxy_set_header On-Behalf-Of ''; + } + location / { add_header Access-Control-Allow-Origin "*" always; add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS" always; @@ -249,6 +333,9 @@ configs: proxy_pass http://linkeddatahub; #proxy_cache backcache; limit_req zone=linked_data burst=30 nodelay; + limit_req zone=sparql burst=10 nodelay; + limit_conn sparql_per_client ${NGINX_SPARQL_CONN_PER_CLIENT:-2}; + limit_conn sparql_total ${NGINX_SPARQL_CONN_TOTAL:-8}; proxy_set_header Host $$host; proxy_set_header X-Forwarded-Host $$host; @@ -321,6 +408,9 @@ configs: proxy_pass http://linkeddatahub; #proxy_cache backcache; limit_req zone=linked_data burst=30 nodelay; + limit_req zone=sparql burst=10 nodelay; + limit_conn sparql_per_client ${NGINX_SPARQL_CONN_PER_CLIENT:-2}; + limit_conn sparql_total ${NGINX_SPARQL_CONN_TOTAL:-8}; proxy_set_header Host $$host; proxy_set_header X-Forwarded-Host $$host; diff --git a/make/local.mk b/make/local.mk index c218bbfe3..20123e898 100644 --- a/make/local.mk +++ b/make/local.mk @@ -15,15 +15,18 @@ sef: release: ./release.sh -# Set rdf/pom.xml and cli/pom.xml to the platform version in pom.xml. Both ship with the platform -# release, so all three versions are kept in step; release.sh runs this around the release version -# bumps, and this target is for drift and for manual SNAPSHOT bumps +# Set rdf/pom.xml, cli/pom.xml and the ldh plugin's plugin.json to the platform version in pom.xml. +# All ship with the platform release, so the versions are kept in step; release.sh runs this around +# the release version bumps, and this target is for drift and for manual SNAPSHOT bumps cli-version: @version=$$(mvn -q help:evaluate -Dexpression=project.version -DforceStdout); \ for project in rdf cli; do \ (cd $$project && mvn -B -q versions:set -DnewVersion="$$version" -DgenerateBackupPoms=false) && \ echo "$$project/pom.xml set to $$version"; \ - done + done; \ + plugin=plugins/ldh/.claude-plugin/plugin.json; \ + jq --arg version "$$version" '.version = $$version' $$plugin > $$plugin.tmp && mv $$plugin.tmp $$plugin && \ + echo "$$plugin set to $$version" # Build the linkeddatahub-rdf library: the vocabularies and the document shapes the API accepts. # Installed rather than packaged because the CLI - and Web-Algebra's ldh-* operations, in their own diff --git a/platform/datasets/admin.trig b/platform/datasets/admin.trig index 1ca1c0f52..5f50ca6c7 100644 --- a/platform/datasets/admin.trig +++ b/platform/datasets/admin.trig @@ -264,6 +264,25 @@ WHERE } +# public dataspace context + + +{ + + a dh:Item ; + sioc:has_container ; + dct:title "Public dataspace context access" ; + foaf:primaryTopic . + + a acl:Authorization ; + rdfs:label "Public dataspace context access" ; + rdfs:comment "Allows non-authenticated read access to the dataspace descriptions, which the client-side stylesheet preloads for every visitor" ; + acl:accessTo ; + acl:mode acl:Read ; + acl:agentClass foaf:Agent, acl:AuthenticatedAgent . + +} + # append authorization request diff --git a/platform/namespace-ontology.trig.template b/platform/namespace-ontology.trig.template index 866390d1e..b90378743 100644 --- a/platform/namespace-ontology.trig.template +++ b/platform/namespace-ontology.trig.template @@ -54,6 +54,25 @@ } +# public dataspace context authorization + + +{ + + a dh:Item ; + sioc:has_container ; + dct:title "Public dataspace context access" ; + foaf:primaryTopic . + + a acl:Authorization ; + rdfs:label "Public dataspace context access" ; + rdfs:comment "Allows non-authenticated read access to the dataspace descriptions, which the client-side stylesheet preloads for every visitor" ; + acl:accessTo <${end_user_origin}/context> ; + acl:mode acl:Read ; + acl:agentClass foaf:Agent, acl:AuthenticatedAgent . + +} + # SPARQL endpoint authorization diff --git a/plugins/ldh/.claude-plugin/plugin.json b/plugins/ldh/.claude-plugin/plugin.json new file mode 100644 index 000000000..166e32764 --- /dev/null +++ b/plugins/ldh/.claude-plugin/plugin.json @@ -0,0 +1,19 @@ +{ + "name": "ldh", + "version": "6.2.0-SNAPSHOT", + "description": "Build Linked Data dataspaces, datasets and applications on LinkedDataHub with the ldh CLI: provision a dataspace, model and import data, declare documents, views and charts, and write reusable packages", + "author": { + "name": "AtomGraph", + "url": "https://atomgraph.com" + }, + "homepage": "https://docs.linkeddatahub.com/", + "repository": "https://github.com/AtomGraph/LinkedDataHub", + "license": "Apache-2.0", + "keywords": [ + "linked-data", + "rdf", + "sparql", + "knowledge-graph", + "linkeddatahub" + ] +} diff --git a/plugins/ldh/README.md b/plugins/ldh/README.md new file mode 100644 index 000000000..d46aeb518 --- /dev/null +++ b/plugins/ldh/README.md @@ -0,0 +1,62 @@ +# ldh — Claude Code plugin for LinkedDataHub + +Skills that teach Claude Code to build on [LinkedDataHub](https://github.com/AtomGraph/LinkedDataHub) with the +`ldh` command line interface: provision a dataspace, model and load data, build an app, and write a reusable +package. + +## Install + +In Claude Code: + +``` +/plugin marketplace add AtomGraph/LinkedDataHub +/plugin install ldh@linkeddatahub +``` + +You need a LinkedDataHub instance (or a checkout to run one from), the owner's WebID certificate, and the `ldh` +CLI (`make cli` in this repository). + +## Skills + +Claude loads a skill on its own when your request matches what the skill covers, so you don't need to name it. +To call one explicitly, use `/ldh:`. + +| Skill | Covers | +|---|---| +| `ldh:dataspace` | Provisioning a new dataspace on a self-hosted instance: the configuration files, the restart, and checking that it is up | +| `ldh:dataset` | The namespace ontology (classes, constructors, constraints, property views) and its install; CSV and RDF imports; federation with remote SPARQL endpoints; building a large dataset standalone from CSV, JSON or XML and bulk-loading it | +| `ldh:app` | Documents, blocks, views and charts, and an app kept as a directory and installed with `ldh push` | +| `ldh:package` | A reusable package: an ontology plus an XSLT stylesheet that customises rendering, published to a registry and imported with `ldh packages add` | + +## Example requests + +Ask the way you would ask a colleague. A request can span several skills, and Claude uses them in turn. + +**Dataspace** + +> Add a dataspace called inventory to my local LinkedDataHub. + +**Dataset** + +> Here are products.csv and categories.csv. Model them with schema.org, import them into the inventory dataspace, +> and make each category's page list its products. + +> This 2 GB XML dump of the company register is too big to import through the API. Turn it into a dataset that +> follows LinkedDataHub's conventions and bulk-load it. + +**App** + +> In this folder, set up a local LinkedDataHub and make a "library" dataspace for a book catalogue from the CSVs +> in ./data (books reference authors by authorId): a page with the books as a table, a page with the authors, a +> chart of books per author, and each author's page showing their books. + +**Package** + +> Write a reusable package that renders schema:ratingValue values as stars, publish it to a package registry +> hosted on my instance, and import it into the inventory dataspace. + +## Versioning + +The plugin's version follows the platform's: `release.sh` and `make cli-version` set +`.claude-plugin/plugin.json` along with the `rdf/` and `cli/` poms. A change to the CLI's options or to the +configuration the skills quote updates the skills in the same commit. diff --git a/plugins/ldh/skills/app/SKILL.md b/plugins/ldh/skills/app/SKILL.md new file mode 100644 index 000000000..b4f52f1f4 --- /dev/null +++ b/plugins/ldh/skills/app/SKILL.md @@ -0,0 +1,237 @@ +--- +name: app +description: Build a LinkedDataHub application declaratively with the ldh CLI — the document tree (containers and items), content blocks, SPARQL views in table/grid/list/map/graph layouts, charts, federated queries, public access, and an app directory installed with ldh push. Use when the task is to present data in an existing dataspace or to package an installable app. For the domain model and data imports use ldh:dataset; for rendering that data cannot declare (custom XSLT) use ldh:package. +--- + +# Build a LinkedDataHub application + +A LinkedDataHub app is **declared, not generated**: it is RDF documents in the dataspace, with no code +and no build step. Pages are documents. A page's content is an ordered list of blocks. Lists, tables, +maps and charts are blocks backed by SPARQL queries. The whole app can live in a directory of Turtle +files that `ldh push` writes into the dataspace, so it is versioned in git and re-installable. + +References: +- HTTP API for agents: https://github.com/AtomGraph/LinkedDataHub/blob/master/AGENTS.md +- CLI (incl. `push`): https://docs.linkeddatahub.com/reference/command-line-interface/ +- Documents and blocks: https://docs.linkeddatahub.com/reference/data-model/ +- Worked examples, one directory per app: + https://github.com/AtomGraph/LinkedDataHub-Apps/tree/master/demo (`northwind-traders`: dashboards, + faceted views, charts; `copenhagen`: map views) + +## Ground rules shared with the other ldh skills + +- **The default graph is empty.** Every query pattern goes inside `GRAPH ?g { … }`. Give each + linked resource its own graph variable, because each resource lives in its own document. +- A document is not the thing it describes. The topic is ``, linked with + `foaf:primaryTopic`, and its properties go there. +- Setup: `export LDH_BASE=https:/// LDH_CERT_FILE= LDH_CERT_PASSWORD=…`. + The password is the deployment's, never a guess. +- **Use an `ldh` that matches the instance.** Build it from the same LinkedDataHub checkout or release + as the running image (`ldh --version`). An instance whose end-user responses carry no + `https://w3id.org/atomgraph/linkeddatahub/dataspaces#admin` `Link` (check with `ldh get --head --accept text/turtle "$LDH_BASE"`) + predates the CLI's admin discovery. There, `ldh admin` commands answer `403` for the end-user base, + and `import ontology --file` refuses it. Use the image built from the same checkout as the CLI. + +## The document model + +| Resource | Shape | +|---|---| +| Container (a section other documents live under) | `<> a dh:Container ; dct:title "…"` | +| Item (a leaf page: a dashboard, a chart page, or a record) | `<> a dh:Item ; dct:title "…"`, plus `foaf:primaryTopic <#this>` when it describes a thing | +| Rich-text block | `<#intro> a ldh:XHTML ; rdf:value """

…
"""^^rdf:XMLLiteral` | +| Embedded resource block | `<#b> a ldh:Object ; rdf:value ` | +| Block order | `<> rdf:_1 <#intro> ; rdf:_2 <#b>` on the document | + +**Only `ldh:XHTML` and `ldh:Object` may be `rdf:_N` values.** Views, charts and anything else are +wrapped in an `ldh:Object` whose `rdf:value` names them. Bare ones are rejected by validation. +XHTML must be well-formed, in the XHTML namespace, with each tag on one line. + +Pick the type by what will live under the page, not by how it looks. A page made only of blocks +(a chart page, a dashboard, an about page) with nothing beneath it is a `dh:Item`. Make it a +`dh:Container` only when documents will be created under it, e.g. the target of an import. Both +types take the same blocks. + +Built-ins worth knowing: `rdf:value ldh:ChildrenView` lists the container's children, and the query +`ldh:SelectChildren` selects them for a view of your own (e.g. a map of the children). + +## Views and charts + +**Views render resources; charts render numbers.** A view's `SELECT` returns resources (their rows are +navigable, facetable and linkable). A query with `COUNT`/`SUM`/`GROUP BY` belongs in a chart, never +a view. + +```turtle +@prefix ldh: . +@prefix ac: . +@prefix dh: . +@prefix rdf: . +@prefix dct: . +@prefix sp: . +@prefix spin: . + +<> a dh:Container ; + dct:title "Products" ; + rdf:_1 <#intro> ; + rdf:_2 <#products-block> ; + rdf:_3 <#per-category-block> . + +<#intro> a ldh:XHTML ; + rdf:value """

Everything in the catalogue.

"""^^rdf:XMLLiteral . + +<#products-block> a ldh:Object ; + dct:title "All products" ; + rdf:value <#products-view> . + +<#products-view> a ldh:View ; + dct:title "All products" ; + spin:query <#select-products> ; + ac:mode ac:TableMode . + +<#select-products> a sp:Select ; + dct:title "Select products" ; + sp:text """PREFIX schema: +SELECT DISTINCT ?product +WHERE { GRAPH ?g { ?product a schema:Product ; schema:name ?name } } +ORDER BY ?name""" . + +<#per-category-block> a ldh:Object ; + dct:title "Products per category" ; + rdf:value <#per-category> . + +<#per-category> a ldh:ResultSetChart ; + dct:title "Products per category" ; + spin:query <#count-per-category> ; + ldh:chartType ac:BarChart ; + ldh:categoryVarName "categoryName" ; + ldh:seriesVarName "productCount" . + +<#count-per-category> a sp:Select ; + dct:title "Count products per category" ; + sp:text """PREFIX schema: +SELECT ?categoryName (COUNT(DISTINCT ?product) AS ?productCount) +WHERE { + GRAPH ?pg { ?product a schema:Product ; schema:category ?category } + GRAPH ?cg { ?category schema:name ?categoryName } +} +GROUP BY ?category ?categoryName +ORDER BY DESC(?productCount)""" . +``` + +- View layout modes (`ac:mode`): `ac:ListMode`, `ac:TableMode`, `ac:GridMode`, `ac:MapMode` (the + resources need `geo:lat`/`geo:long`), `ac:GraphMode`. +- Chart types (`ldh:chartType`): `ac:BarChart`, `ac:LineChart`, `ac:ScatterChart`. + `ldh:categoryVarName` and `ldh:seriesVarName` name the query's variables, without `?`. +- A query runs against the dataspace's own endpoint unless it names a service: + `<#q> ldh:service <#dbpedia>` with `<#dbpedia> a sd:Service ; dct:title "DBpedia" ; + sd:endpoint ; sd:supportedLanguage sd:SPARQL11Query`. Time a remote + query before building on it, and write it against the remote store's own graph layout. +- Every non-optional pattern filters rows. A pattern that is there only to sort or to display + (`?book schema:datePublished ?year` for `ORDER BY ?year`) goes in `OPTIONAL`, or resources missing + that value silently drop out of the view. +- Per-resource panels ("orders of this customer") are property views in the ontology (ldh:dataset), + not blocks. + +## The app as a directory + +`ldh push` maps a directory onto the document tree: + +``` +my-app/ +├── root.ttl → PUT to $LDH_BASE (the root document itself) +├── products.ttl → PUT to ${LDH_BASE}products/ +├── products/ +│ ├── products.csv → uploaded into products/ (unless ignored) +│ └── featured.ttl → PUT to ${LDH_BASE}products/featured/ +├── admin/model/ns.ttl → the ontology, installed separately (ldh:dataset), so ignored +└── .ldhignore → gitignore-style: admin/ *.sh *.csv +``` + +Relative URIs in each file resolve against its target URL, so write `<>` and `<#…>`. A +subdirectory's document `name.ttl` sits beside it, not inside it. Non-RDF files are uploaded and +addressed as `${LDH_BASE}uploads/{sha1}`, and `.xml` files count as RDF/XML. Ignore CSVs that +`ldh import csv` uploads itself. + +**`root.ttl` replaces the seeded root document**, so keep its type: read it first +(`ldh get --accept text/turtle "$LDH_BASE"`) and start `root.ttl` from that. A push replaces every +document it writes, so a re-push converges. It stops at the first failed request, and stdout lists +what was written before it. + +```bash +ldh push --dry-run --dir my-app "$LDH_BASE" # parses every file, prints the plan, sends nothing +ldh push --dir my-app "$LDH_BASE" +``` + +### Install order + +An installable app is an `install.sh` that runs, in this order: + +1. `ldh admin make-public` (only if anonymous visitors should read it). Each run adds another + authorization, so run it once. +2. Install the namespace ontology and any `ldh packages add` (ldh:dataset). +3. `ldh push --dir "$app_dir" "$LDH_BASE"`, which creates the containers the imports write into. +4. `ldh import csv|rdf` per data file (ldh:dataset), then wait for each import to finish. + +**The script is self-contained.** It needs `ldh` and nothing else, so it runs from wherever the app +directory is copied. The demo apps in LinkedDataHub-Apps source a shared `lib/ldh-app.sh`, which is +internal to that repository. Do not copy that `source` line, and do not reference any other checkout +by path. + +```bash +#!/usr/bin/env bash +# Installs the app with the ldh CLI. Reads LDH_BASE, LDH_CERT_FILE, LDH_CERT_PASSWORD (and LDH_PROXY). +set -euo pipefail +: "${LDH_BASE:?}" "${LDH_CERT_FILE:?}" "${LDH_CERT_PASSWORD:?}" + +app_dir="$(cd "$(dirname "$0")" && pwd)" + +ldh admin make-public +ldh admin import ontology --file "$app_dir/admin/model/ns.ttl" # ldh:dataset +ldh push --dir "$app_dir" "$LDH_BASE" + +# the target document the import is added to; --wait returns once the data is in, and fails if it is not +ldh import csv --title "Books" --query-file "$app_dir/books/books.rq" \ + --csv-file "$app_dir/books/books.csv" --wait "${LDH_BASE}books/" +``` + +Every command takes the end-user base. The admin commands find the admin app from it, so the script +never derives an `admin.` URI. Keep `admin/`, `*.sh`, `*.rq` and `*.csv` in `.ldhignore` so `push` +does not upload them. + +Finer access control: `ldh admin create group`, `ldh admin add agent`, `ldh admin create authorization` +(https://docs.linkeddatahub.com/reference/administration/acl/). + +## Building incrementally instead + +For one-off changes to a live dataspace, the `create`/`add` commands each make one request and print +the new URL: + +```bash +products=$(ldh create container --parent "$LDH_BASE" --title "Products" --slug products) +ldh add select --uri "${products}#select-products" --title "Select products" \ + --query-file "$PWD/select-products.rq" "$products" +ldh add view --uri "${products}#products-view" --title "All products" \ + --query "${products}#select-products" --mode https://w3id.org/atomgraph/client#TableMode "$products" +ldh add object-block --value "${products}#products-view" --title "All products" "$products" +ldh add xhtml-block --value '

…

' "$products" +ldh add result-set-chart --query + + + + + + + + + + + +
+
+
+