Dev

.github/workflows/biosim_extractor_project-ci.yaml → .github/workflows/biosimdb_interface_project-ci.yaml

@@ -1,4 +1,4 @@
-name: biosim-extractor CI
+name: biosimdb-interface CI
 
 on:
   push:
@@ -25,16 +25,38 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install biosim-extractor and its testing dependencies
+      - name: Install biosimdb-interface and its testing dependencies
         run: pip install -e .[testing]
 
+      - name: Get latest biosim-schema release tag (or fail if none)
+        id: schema-tag
+        shell: bash
+        run: |
+          # Attempt to get the latest release tag. If no releases exist, this command will fail,
+          # and the workflow will stop here, which is appropriate if release assets are mandatory.
+          TAG=$(gh release view --repo CCPBioSim/biosim-schema --json tagName --template '{{.tagName}}')
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download biosim-schema artifacts
+        run: gh release download ${{ steps.schema-tag.outputs.tag }} --repo CCPBioSim/biosim-schema --pattern "biosim-schema-artifacts.tar.gz"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract biosim-schema artifacts
+        shell: bash
+        run: |
+          mkdir -p biosim-schema
+          tar -xzf biosim-schema-artifacts.tar.gz -C biosim-schema
+
       - name: Run test suite
-        run: pytest --cov biosim_extractor --cov-report term-missing --cov-append .
+        run: pytest --cov biosimdb_interface --cov-report term-missing --cov-append tests/
         env:
-          WEBFORM_SCHEMA_PATH: ""
-          ENGINE_MAPPING_SCHEMA_PATH: ""
-          BIOSIM_SCHEMA_PATH: ""
-          SECRET_KEY: "test-secret-key"
+          WEBFORM_SCHEMA_PATH: ${{ github.workspace }}/biosim-schema/project/schema_webformfields.json
+          ENGINE_MAPPING_SCHEMA_PATH: ${{ github.workspace }}/biosim-schema/project/schema_enginemappings.json
+          BIOSIM_SCHEMA_PATH: ${{ github.workspace }}/biosim-schema/biosim_schema/schema/biosim_schema.yaml
+          SECRET_KEY: "ci-testing-only-key"
 
       - name: Coveralls GitHub Action
         uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e # v2.3.7
@@ -43,6 +65,19 @@ jobs:
           parallel: true
           fail-on-error: false
 
+  coveralls-finish:
+    name: Finish Coveralls
+    needs: tests
+    if: ${{ always() }}
+    runs-on: ubuntu-24.04
+    steps:
+    - name: Coveralls Finished
+      uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e # v2.3.7
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        parallel-finished: true
+        fail-on-error: false
+
   docs:
     runs-on: ubuntu-latest
     timeout-minutes: 15

.github/workflows/release.yaml

@@ -28,7 +28,7 @@ jobs:
     - name: Get latest release from pip
       id: latestreleased
       run: |
-        PREVIOUS_VERSION=$(python -m pip index versions CCPBioSim-Python-Template | grep "CCPBioSim-Python-Template" | cut -d "(" -f2 | cut -d ")" -f1)
+        PREVIOUS_VERSION=$(python -m pip index versions biosimdb-interface | grep "biosimdb-interface" | cut -d "(" -f2 | cut -d ")" -f1)
         echo "pip_tag=$PREVIOUS_VERSION" >> "$GITHUB_OUTPUT"
         echo $PREVIOUS_VERSION
 
@@ -48,14 +48,19 @@ jobs:
       - name: checkout
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
 
-      - name: Change version in repo and CITATION.cff
+      - name: Change version in repo (init and schema) and CITATION.cff
         run: |
+          VERSION="${{ github.event.inputs.version }}"
+
           # Update Python package version
-          sed -i "s/__version__ =.*/__version__ = \"${{ github.event.inputs.version }}\"/g" biosimdb-interface/biosimdb_interface/__init__.py
+          sed -i "s/^__version__ = .*/__version__ = \"${VERSION}\"/g" biosim_schema/__init__.py
+
+          # Keep Sphinx release metadata in sync
+          sed -i -E "s/^(release = ).*/\1'${VERSION}'/" docs/source/conf.py
 
           # Update CITATION.cff version and date-released
           if [ -f CITATION.cff ]; then
-            sed -i -E "s/^(version:\s*).*/\1${{ github.event.inputs.version }}/" CITATION.cff
+            sed -i -E "s/^(version:\s*).*/\1${VERSION}/" CITATION.cff
             sed -i -E "s/^(date-released:\s*).*/\1'$(date -u +%F)'/" CITATION.cff
           fi
 
@@ -69,6 +74,7 @@ jobs:
           body: |
             Update version
             - Update the __init__.py with new release
+            - Update docs/source/conf.py release
             - Update CITATION.cff version & date-released
             - Auto-generated by [CI]
           committer: version-updater <vu.bot@users.noreply.github.com>
@@ -118,31 +124,3 @@ jobs:
           name: v${{ github.event.inputs.version }}
           generate_release_notes: true
           tag_name: ${{ github.event.inputs.version }}
-
-  pypi:
-    name: make pypi release
-    needs: [tag, release]
-    runs-on: ubuntu-24.04
-    steps:
-
-    - name: checkout
-      uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
-      with:
-        ref: main
-
-    - name: Set up Python
-      uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
-      with:
-        python-version: 3.14.0
-
-    - name: Install flit
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install flit~=3.9
-
-    - name: Build and publish
-      run: |
-        flit publish
-      env:
-        FLIT_USERNAME: __token__
-        FLIT_PASSWORD:  ${{ secrets.PYPI_API_TOKEN }}

README.md

@@ -6,7 +6,6 @@ A repository for extracting and uploading simulation data via a web interface to
 
 | Category       | Badges |
 |----------------|--------|
-| **Build**      | [![CI](https://github.com/CCPBioSim/biosimdb-interface/actions/workflows/example_package_project-ci.yaml/badge.svg)](https://github.com/CCPBioSim/biosimdb-interface/actions/workflows/example_package_project-ci.yaml) |
-| **Documentation** | [![Docs – Status](https://app.readthedocs.org/projects/ccpbiosim-biosimdb-interface/badge/?version=latest)](https://ccpbiosim-biosimdb-interface.readthedocs.io/en/latest/) |
-| **PyPI**       | [![PyPI - Version](https://img.shields.io/pypi/v/CCPBioSim-biosimdb-interface.svg)](https://pypi.org/project/CCPBioSim-biosimdb-interface/) [![PyPI - Status](https://img.shields.io/pypi/status/CCPBioSim-biosimdb-interface.svg)](https://pypi.org/project/CCPBioSim-biosimdb-interface/) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/CCPBioSim-biosimdb-interface.svg)](https://pypi.org/project/CCPBioSim-biosimdb-interface/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/CCPBioSim-biosimdb-interface.svg)](https://pypi.org/project/CCPBioSim-biosimdb-interface/) |
+| **Build**      | [![CI](https://github.com/CCPBioSim/biosimdb-interface/actions/workflows/biosimdb_interface_project-ci.yaml/badge.svg)](https://github.com/CCPBioSim/biosimdb-interface/actions/workflows/example_package_project-ci.yaml) |
+| **Documentation** | [![Docs – Status](https://app.readthedocs.org/projects/biosimdb-interface/badge/?version=latest)](https://biosimdb-interface.readthedocs.io/en/latest/) |
 | **Quality**    | [![Coverage Status](https://coveralls.io/repos/github/CCPBioSim/biosimdb-interface/badge.svg?branch=main)](https://coveralls.io/github/CCPBioSim/biosimdb-interface?branch=main) |

biosimdb_interface/form/extract.py

@@ -11,11 +11,41 @@
 import tempfile
 
 from biosim_extractor.metadata.populatemetadata import MetadataPopulator
-from flask import jsonify, request
+from flask import jsonify, request, session
 
 from . import form_bp
 
 
+def extract_files_validate(top_file, traj_file):
+    """Extract metadata from simulation files and validate against the schema.
+
+    Args:
+        top_file (str): Path to the topology file.
+        traj_file (str or list[str]): Path or list of paths to the trajectory file(s).
+
+    Returns:
+        tuple: A tuple containing:
+            - result (dict): The extracted and populated metadata dictionary.
+            - validation_errors (list[str]): A list of validation error messages,
+              empty if validation succeeded.
+    """
+    populator = MetadataPopulator(
+        schema_path=os.getenv("ENGINE_MAPPING_SCHEMA_PATH", ""),
+        top_file=top_file,
+        traj_file=traj_file,
+    )
+
+    result = populator.populate()
+    biosimschema_path = os.getenv("BIOSIM_SCHEMA_PATH", "")
+    validation_errors = []
+    try:
+        populator.validate(result, biosimschema_path, strict=True)
+    except ValueError as e:
+        validation_errors = str(e).splitlines()
+
+    return result, validation_errors
+
+
 @form_bp.route("/extract_metadata", methods=["POST"])
 def extract_metadata():
     """Extract simulation metadata from uploaded topology and trajectory files.
@@ -40,33 +70,20 @@ def extract_metadata():
         if not topology or not trajectories:
             return jsonify({"error": "Simulation files are missing."}), 400
 
-        with (
-            tempfile.NamedTemporaryFile(
-                suffix=os.path.splitext(topology.filename)[1]
-            ) as topo_file,
-            tempfile.TemporaryDirectory() as temp_dir,
-        ):
-            topology.save(topo_file.name)
+        with tempfile.TemporaryDirectory() as temp_dir:
+            topo_path = os.path.join(temp_dir, topology.filename)
+            topology.save(topo_path)
             traj_files = []
 
             for traj in trajectories:
                 traj_path = os.path.join(temp_dir, traj.filename)
                 traj.save(traj_path)
                 traj_files.append(traj_path)
 
-            populator = MetadataPopulator(
-                schema_path=os.getenv("ENGINE_MAPPING_SCHEMA_PATH", ""),
-                top_file=topo_file.name,
-                traj_file=traj_files,
-            )
-
-            result = populator.populate()
-            biosimschema_path = os.getenv("BIOSIM_SCHEMA_PATH", "")
-            validation_errors = []
-            try:
-                populator.validate(result, biosimschema_path, strict=True)
-            except ValueError as e:
-                validation_errors = str(e).splitlines()
+            result, validation_errors = extract_files_validate(topo_path, traj_files)
+
+            # Keep authoritative extracted payload on the server
+            session["extracted_metadata"] = result
 
             if len(validation_errors) > 0:
                 return jsonify(

biosimdb_interface/form/webform.py

@@ -51,6 +51,13 @@ def webform():
             # include file info in output, ro-crate?
             json_form = form_to_json(request.form)
             json_form = remove_empty_fields(json_form)
+
+            # convert to standard units
+            json_form = convert_populated_metadata_units(json_form)
+
+            # NOTE: note used yet, could be used to validate extracted fields are matching what is returned from json_form
+            # extracted = session.get("extracted_metadata")
+
             biosimschema_path = os.getenv("BIOSIM_SCHEMA_PATH", "")
 
             validation_errors = []
@@ -66,9 +73,6 @@ def webform():
                     }
                 )
 
-            # convert to standard units
-            json_form = convert_populated_metadata_units(json_form)
-
             if action == "submit":
                 save_pending_submission()
                 if not token:

biosimdb_interface/schema/helpers.py

@@ -6,29 +6,6 @@
 import argparse
 import json
 
-
-def load_schema(self):
-    """Load schema JSON from a URL or local file path into ``self.schema``.
-
-    Args:
-        self: Object with ``schema_path`` (str) attribute.
-
-    Returns:
-        dict: Parsed JSON schema.
-    """
-    if self.schema_path.startswith("http://") or self.schema_path.startswith(
-        "https://"
-    ):
-        import urllib.request
-
-        with urllib.request.urlopen(self.schema_path) as f:
-            self.schema = json.load(f)
-    else:
-        with open(self.schema_path) as f:
-            self.schema = json.load(f)
-    return self.schema
-
-
 # -----------------------------
 # Main class
 # -----------------------------
@@ -50,9 +27,16 @@ def load_schema(self):
         Returns:
             dict: Parsed JSON schema.
         """
-        with open(self.schema_path) as f:
-            self.schema = json.load(f)
-            return self.schema
+        if self.schema_path.startswith(("http://", "https://")):
+            import urllib.request
+
+            with urllib.request.urlopen(self.schema_path) as f:
+                self.schema = json.load(f)
+        else:
+            with open(self.schema_path) as f:
+                self.schema = json.load(f)
+
+        return self.schema
 
 
 # -----------------------------

biosimdb_interface/schema/webform.py

@@ -30,7 +30,11 @@ def get_simulation_metadata():
     Returns:
         dict: Parsed simulation metadata schema.
     """
-    path = os.getenv("WEBFORM_SCHEMA_PATH", "")
+    path = os.getenv("WEBFORM_SCHEMA_PATH")
+    if not path:
+        # Return an empty schema or raise a more descriptive error if preferred
+        return {}
+
     mtime = os.path.getmtime(path)
     if _cache["mtime"] != mtime:
         _cache["schema"] = SchemaPopulator(schema_path=path).load_schema()

biosimdb_interface/templates/macros/webform.html

@@ -51,6 +51,7 @@
     {% set key = group_name ~ '[' ~ field_name ~ ']' %}
     {% set value = form_data.get(key, '') %}
     {% set error = errors.get(key) %}
+    {% set is_locked = field.get("extracted_only") or field.get("readonly") %}
 
 
     {% if field.type == "textarea" %}
@@ -91,13 +92,15 @@
         <textarea class="form-control form-control-sm {% if error %}is-invalid{% endif %}"
                 name="{{ key }}"
                 rows="2"
+                {% if is_locked %}readonly{% endif %}
                 {% if field.placeholder %}placeholder="{{ field.placeholder }}"{% endif %}
                 {% if field.required %}required{% endif %}>{{ value }}</textarea>
     {% elif field.type == "select" %}
         {% set selected_values = value if value is iterable and value is not string else ([value] if value else []) %}
         <select class="form-select form-select-sm {% if error %}is-invalid{% endif %}"
                 name="{{ key }}{% if field.multiple %}[]{% endif %}"
                 {% if field.multiple %}multiple size="{{ [field.options|length + 1, 3] | min }}"{% endif %}
+                {% if is_locked %}disabled{% endif %}
                 {% if field.required %}required{% endif %}>
             <option value=""
                     {% if not field.multiple and (value == '' or value is none) %}selected{% endif %}>
@@ -132,6 +135,7 @@
                 class="form-control form-control-sm {% if error %}is-invalid{% endif %}"
                 value="{{ value }}"
                 name="{{ key }}"
+                {% if is_locked %}readonly{% endif %}
                 {% if field.placeholder %}placeholder="{{ field.placeholder }}"{% endif %}
                 {% if field.required %}required{% endif %}>
         {% endif %}
@@ -146,6 +150,10 @@
         <button type="button" class="btn btn-sm btn-outline-secondary w-100 add-field mt-1" data-field="{{ field_name }}">Add Another {{ field.label }}</button>
     {% endif %}
 
+    {% if is_locked %}
+    <small class="text-muted">Extracted from files{% if field.get("source") %} ({{ field.source }}){% endif %}</small>
+    {% endif %}
+
     {% if depends %}
     </div>
     {% endif %}

pyproject.toml

@@ -41,7 +41,7 @@ dependencies = [
     "MDAnalysis==2.10.0",
     "python-dotenv==1.2.2",
     "gunicorn>=23,<24",
-    "biosim-extractor @ git+ssh://git@github.com/CCPBioSim/biosim-extractor.git"
+    "biosim-extractor>=0.0.2,<1.0"
 ]
 
 [project.urls]