diff --git a/.github/workflows/biosim_extractor_project-ci.yaml b/.github/workflows/biosimdb_interface_project-ci.yaml similarity index 52% rename from .github/workflows/biosim_extractor_project-ci.yaml rename to .github/workflows/biosimdb_interface_project-ci.yaml index 83bbddd..c0b3bfc 100644 --- a/.github/workflows/biosim_extractor_project-ci.yaml +++ b/.github/workflows/biosimdb_interface_project-ci.yaml @@ -1,4 +1,4 @@ -name: biosim-extractor CI +name: biosimdb-interface CI on: push: @@ -25,16 +25,38 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install biosim-extractor and its testing dependencies + - name: Install biosimdb-interface and its testing dependencies run: pip install -e .[testing] + - name: Get latest biosim-schema release tag (or fail if none) + id: schema-tag + shell: bash + run: | + # Attempt to get the latest release tag. If no releases exist, this command will fail, + # and the workflow will stop here, which is appropriate if release assets are mandatory. + TAG=$(gh release view --repo CCPBioSim/biosim-schema --json tagName --template '{{.tagName}}') + echo "tag=$TAG" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Download biosim-schema artifacts + run: gh release download ${{ steps.schema-tag.outputs.tag }} --repo CCPBioSim/biosim-schema --pattern "biosim-schema-artifacts.tar.gz" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract biosim-schema artifacts + shell: bash + run: | + mkdir -p biosim-schema + tar -xzf biosim-schema-artifacts.tar.gz -C biosim-schema + - name: Run test suite - run: pytest --cov biosim_extractor --cov-report term-missing --cov-append . + run: pytest --cov biosimdb_interface --cov-report term-missing --cov-append tests/ env: - WEBFORM_SCHEMA_PATH: "" - ENGINE_MAPPING_SCHEMA_PATH: "" - BIOSIM_SCHEMA_PATH: "" - SECRET_KEY: "test-secret-key" + WEBFORM_SCHEMA_PATH: ${{ github.workspace }}/biosim-schema/project/schema_webformfields.json + ENGINE_MAPPING_SCHEMA_PATH: ${{ github.workspace }}/biosim-schema/project/schema_enginemappings.json + BIOSIM_SCHEMA_PATH: ${{ github.workspace }}/biosim-schema/biosim_schema/schema/biosim_schema.yaml + SECRET_KEY: "ci-testing-only-key" - name: Coveralls GitHub Action uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e # v2.3.7 @@ -43,6 +65,19 @@ jobs: parallel: true fail-on-error: false + coveralls-finish: + name: Finish Coveralls + needs: tests + if: ${{ always() }} + runs-on: ubuntu-24.04 + steps: + - name: Coveralls Finished + uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e # v2.3.7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + parallel-finished: true + fail-on-error: false + docs: runs-on: ubuntu-latest timeout-minutes: 15 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a62fdab..7173036 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -28,7 +28,7 @@ jobs: - name: Get latest release from pip id: latestreleased run: | - PREVIOUS_VERSION=$(python -m pip index versions CCPBioSim-Python-Template | grep "CCPBioSim-Python-Template" | cut -d "(" -f2 | cut -d ")" -f1) + PREVIOUS_VERSION=$(python -m pip index versions biosimdb-interface | grep "biosimdb-interface" | cut -d "(" -f2 | cut -d ")" -f1) echo "pip_tag=$PREVIOUS_VERSION" >> "$GITHUB_OUTPUT" echo $PREVIOUS_VERSION @@ -48,14 +48,19 @@ jobs: - name: checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 - - name: Change version in repo and CITATION.cff + - name: Change version in repo (init and schema) and CITATION.cff run: | + VERSION="${{ github.event.inputs.version }}" + # Update Python package version - sed -i "s/__version__ =.*/__version__ = \"${{ github.event.inputs.version }}\"/g" biosimdb-interface/biosimdb_interface/__init__.py + sed -i "s/^__version__ = .*/__version__ = \"${VERSION}\"/g" biosim_schema/__init__.py + + # Keep Sphinx release metadata in sync + sed -i -E "s/^(release = ).*/\1'${VERSION}'/" docs/source/conf.py # Update CITATION.cff version and date-released if [ -f CITATION.cff ]; then - sed -i -E "s/^(version:\s*).*/\1${{ github.event.inputs.version }}/" CITATION.cff + sed -i -E "s/^(version:\s*).*/\1${VERSION}/" CITATION.cff sed -i -E "s/^(date-released:\s*).*/\1'$(date -u +%F)'/" CITATION.cff fi @@ -69,6 +74,7 @@ jobs: body: | Update version - Update the __init__.py with new release + - Update docs/source/conf.py release - Update CITATION.cff version & date-released - Auto-generated by [CI] committer: version-updater @@ -118,31 +124,3 @@ jobs: name: v${{ github.event.inputs.version }} generate_release_notes: true tag_name: ${{ github.event.inputs.version }} - - pypi: - name: make pypi release - needs: [tag, release] - runs-on: ubuntu-24.04 - steps: - - - name: checkout - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 - with: - ref: main - - - name: Set up Python - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 - with: - python-version: 3.14.0 - - - name: Install flit - run: | - python -m pip install --upgrade pip - python -m pip install flit~=3.9 - - - name: Build and publish - run: | - flit publish - env: - FLIT_USERNAME: __token__ - FLIT_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} diff --git a/README.md b/README.md index ecf8ee5..349f9b5 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ A repository for extracting and uploading simulation data via a web interface to | Category | Badges | |----------------|--------| -| **Build** | [![CI](https://github.com/CCPBioSim/biosimdb-interface/actions/workflows/example_package_project-ci.yaml/badge.svg)](https://github.com/CCPBioSim/biosimdb-interface/actions/workflows/example_package_project-ci.yaml) | -| **Documentation** | [![Docs – Status](https://app.readthedocs.org/projects/ccpbiosim-biosimdb-interface/badge/?version=latest)](https://ccpbiosim-biosimdb-interface.readthedocs.io/en/latest/) | -| **PyPI** | [![PyPI - Version](https://img.shields.io/pypi/v/CCPBioSim-biosimdb-interface.svg)](https://pypi.org/project/CCPBioSim-biosimdb-interface/) [![PyPI - Status](https://img.shields.io/pypi/status/CCPBioSim-biosimdb-interface.svg)](https://pypi.org/project/CCPBioSim-biosimdb-interface/) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/CCPBioSim-biosimdb-interface.svg)](https://pypi.org/project/CCPBioSim-biosimdb-interface/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/CCPBioSim-biosimdb-interface.svg)](https://pypi.org/project/CCPBioSim-biosimdb-interface/) | +| **Build** | [![CI](https://github.com/CCPBioSim/biosimdb-interface/actions/workflows/biosimdb_interface_project-ci.yaml/badge.svg)](https://github.com/CCPBioSim/biosimdb-interface/actions/workflows/example_package_project-ci.yaml) | +| **Documentation** | [![Docs – Status](https://app.readthedocs.org/projects/biosimdb-interface/badge/?version=latest)](https://biosimdb-interface.readthedocs.io/en/latest/) | | **Quality** | [![Coverage Status](https://coveralls.io/repos/github/CCPBioSim/biosimdb-interface/badge.svg?branch=main)](https://coveralls.io/github/CCPBioSim/biosimdb-interface?branch=main) | diff --git a/biosimdb_interface/form/extract.py b/biosimdb_interface/form/extract.py index 9417650..71f0c01 100644 --- a/biosimdb_interface/form/extract.py +++ b/biosimdb_interface/form/extract.py @@ -11,11 +11,41 @@ import tempfile from biosim_extractor.metadata.populatemetadata import MetadataPopulator -from flask import jsonify, request +from flask import jsonify, request, session from . import form_bp +def extract_files_validate(top_file, traj_file): + """Extract metadata from simulation files and validate against the schema. + + Args: + top_file (str): Path to the topology file. + traj_file (str or list[str]): Path or list of paths to the trajectory file(s). + + Returns: + tuple: A tuple containing: + - result (dict): The extracted and populated metadata dictionary. + - validation_errors (list[str]): A list of validation error messages, + empty if validation succeeded. + """ + populator = MetadataPopulator( + schema_path=os.getenv("ENGINE_MAPPING_SCHEMA_PATH", ""), + top_file=top_file, + traj_file=traj_file, + ) + + result = populator.populate() + biosimschema_path = os.getenv("BIOSIM_SCHEMA_PATH", "") + validation_errors = [] + try: + populator.validate(result, biosimschema_path, strict=True) + except ValueError as e: + validation_errors = str(e).splitlines() + + return result, validation_errors + + @form_bp.route("/extract_metadata", methods=["POST"]) def extract_metadata(): """Extract simulation metadata from uploaded topology and trajectory files. @@ -40,13 +70,9 @@ def extract_metadata(): if not topology or not trajectories: return jsonify({"error": "Simulation files are missing."}), 400 - with ( - tempfile.NamedTemporaryFile( - suffix=os.path.splitext(topology.filename)[1] - ) as topo_file, - tempfile.TemporaryDirectory() as temp_dir, - ): - topology.save(topo_file.name) + with tempfile.TemporaryDirectory() as temp_dir: + topo_path = os.path.join(temp_dir, topology.filename) + topology.save(topo_path) traj_files = [] for traj in trajectories: @@ -54,19 +80,10 @@ def extract_metadata(): traj.save(traj_path) traj_files.append(traj_path) - populator = MetadataPopulator( - schema_path=os.getenv("ENGINE_MAPPING_SCHEMA_PATH", ""), - top_file=topo_file.name, - traj_file=traj_files, - ) - - result = populator.populate() - biosimschema_path = os.getenv("BIOSIM_SCHEMA_PATH", "") - validation_errors = [] - try: - populator.validate(result, biosimschema_path, strict=True) - except ValueError as e: - validation_errors = str(e).splitlines() + result, validation_errors = extract_files_validate(topo_path, traj_files) + + # Keep authoritative extracted payload on the server + session["extracted_metadata"] = result if len(validation_errors) > 0: return jsonify( diff --git a/biosimdb_interface/form/webform.py b/biosimdb_interface/form/webform.py index 297eda0..52931cc 100644 --- a/biosimdb_interface/form/webform.py +++ b/biosimdb_interface/form/webform.py @@ -51,6 +51,13 @@ def webform(): # include file info in output, ro-crate? json_form = form_to_json(request.form) json_form = remove_empty_fields(json_form) + + # convert to standard units + json_form = convert_populated_metadata_units(json_form) + + # NOTE: note used yet, could be used to validate extracted fields are matching what is returned from json_form + # extracted = session.get("extracted_metadata") + biosimschema_path = os.getenv("BIOSIM_SCHEMA_PATH", "") validation_errors = [] @@ -66,9 +73,6 @@ def webform(): } ) - # convert to standard units - json_form = convert_populated_metadata_units(json_form) - if action == "submit": save_pending_submission() if not token: diff --git a/biosimdb_interface/schema/helpers.py b/biosimdb_interface/schema/helpers.py index 2ef728a..4018e56 100644 --- a/biosimdb_interface/schema/helpers.py +++ b/biosimdb_interface/schema/helpers.py @@ -6,29 +6,6 @@ import argparse import json - -def load_schema(self): - """Load schema JSON from a URL or local file path into ``self.schema``. - - Args: - self: Object with ``schema_path`` (str) attribute. - - Returns: - dict: Parsed JSON schema. - """ - if self.schema_path.startswith("http://") or self.schema_path.startswith( - "https://" - ): - import urllib.request - - with urllib.request.urlopen(self.schema_path) as f: - self.schema = json.load(f) - else: - with open(self.schema_path) as f: - self.schema = json.load(f) - return self.schema - - # ----------------------------- # Main class # ----------------------------- @@ -50,9 +27,16 @@ def load_schema(self): Returns: dict: Parsed JSON schema. """ - with open(self.schema_path) as f: - self.schema = json.load(f) - return self.schema + if self.schema_path.startswith(("http://", "https://")): + import urllib.request + + with urllib.request.urlopen(self.schema_path) as f: + self.schema = json.load(f) + else: + with open(self.schema_path) as f: + self.schema = json.load(f) + + return self.schema # ----------------------------- diff --git a/biosimdb_interface/schema/webform.py b/biosimdb_interface/schema/webform.py index 558edf3..b5998c0 100644 --- a/biosimdb_interface/schema/webform.py +++ b/biosimdb_interface/schema/webform.py @@ -30,7 +30,11 @@ def get_simulation_metadata(): Returns: dict: Parsed simulation metadata schema. """ - path = os.getenv("WEBFORM_SCHEMA_PATH", "") + path = os.getenv("WEBFORM_SCHEMA_PATH") + if not path: + # Return an empty schema or raise a more descriptive error if preferred + return {} + mtime = os.path.getmtime(path) if _cache["mtime"] != mtime: _cache["schema"] = SchemaPopulator(schema_path=path).load_schema() diff --git a/biosimdb_interface/templates/macros/webform.html b/biosimdb_interface/templates/macros/webform.html index dc72bbd..28d223f 100644 --- a/biosimdb_interface/templates/macros/webform.html +++ b/biosimdb_interface/templates/macros/webform.html @@ -51,6 +51,7 @@ {% set key = group_name ~ '[' ~ field_name ~ ']' %} {% set value = form_data.get(key, '') %} {% set error = errors.get(key) %} + {% set is_locked = field.get("extracted_only") or field.get("readonly") %} {% if field.type == "textarea" %} @@ -91,6 +92,7 @@ {% elif field.type == "select" %} @@ -98,6 +100,7 @@