feat: replace mypy and pytype with pyrefly for static type analysis

.github/workflows/cron.yml

@@ -214,7 +214,11 @@ jobs:
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
         ngc --version
+<<<<<<< HEAD
+        BUILD_MONAI=1 ./runtests.sh --build --coverage --pyrefly --unittests --disttests  # unit tests with pyrefly checks, coverage report
+=======
         BUILD_MONAI=1 ./runtests.sh --build --coverage --unittests --disttests  # unit tests with pytype checks, coverage report
+>>>>>>> upstream/dev
         BUILD_MONAI=1 ./runtests.sh --build --coverage --net  # integration tests with coverage report
         coverage xml --ignore-errors
         if pgrep python; then pkill python; fi

.github/workflows/pythonapp.yml

@@ -0,0 +1,244 @@
+# Jenkinsfile.monai-premerge
+name: premerge
+
+on:
+  # quick tests for pull requests and the releasing branches
+  push:
+    branches:
+      - dev
+      - main
+      - releasing/*
+  pull_request:
+    head_ref-ignore:
+      - dev
+
+concurrency:
+  # automatically cancel the previously triggered workflows when there's a newer version
+  group: build-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # caching of these jobs:
+  #   - docker-py3-pip- (shared)
+  #   - ubuntu py37 pip-
+  #   - os-latest-pip- (shared)
+  flake8-py3:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        opt: ["codeformat", "pyrefly"]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+    - name: cache weekly timestamp
+      id: pip-cache
+      run: |
+        echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT
+    - name: cache for pip
+      uses: actions/cache@v4
+      id: cache
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ steps.pip-cache.outputs.datew }}
+    - name: Install dependencies
+      run: |
+        find  /opt/hostedtoolcache/* -maxdepth 0 ! -name 'Python' -exec rm -rf {} \;
+        python -m pip install --upgrade pip wheel
+        python -m pip install -r requirements-dev.txt
+    - name: Lint and type check
+      run: |
+        # clean up temporary files
+        $(pwd)/runtests.sh --build --clean
+        $(pwd)/runtests.sh --build --${{ matrix.opt }}
+
+  quick-py3:  # full dependencies installed tests for different OS
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest, macOS-latest, ubuntu-latest]
+    timeout-minutes: 120
+    steps:
+    - if: runner.os == 'windows'
+      name: Config pagefile (Windows only)
+      uses: al-cheb/configure-pagefile-action@v1.4
+      with:
+        minimum-size: 8GB
+        maximum-size: 16GB
+        disk-root: "D:"
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+    - name: Prepare pip wheel
+      run: |
+        which python
+        python -m pip install --upgrade pip wheel
+    - name: cache weekly timestamp
+      id: pip-cache
+      run: |
+        echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT
+        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+      shell: bash
+    - name: cache for pip
+      uses: actions/cache@v4
+      id: cache
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: ${{ matrix.os }}-latest-pip-${{ steps.pip-cache.outputs.datew }}
+    - if: runner.os == 'windows'
+      name: Install torch cpu from pytorch.org (Windows only)
+      run: |
+        python -m pip install torch==2.4.1 torchvision==0.19.1+cpu --index-url https://download.pytorch.org/whl/cpu
+    - if: runner.os == 'Linux'
+      name: Install itk pre-release (Linux only)
+      run: |
+        python -m pip install --pre -U itk
+        find  /opt/hostedtoolcache/* -maxdepth 0 ! -name 'Python' -exec rm -rf {} \;
+    - name: Install the dependencies
+      run: |
+        python -m pip install --user --upgrade pip wheel
+        python -m pip install torch==2.4.1 torchvision==0.19.1
+        cat "requirements-dev.txt"
+        python -m pip install -r requirements-dev.txt
+        python -m pip list
+        python setup.py develop  # test no compile installation
+      shell: bash
+    - name: Run compiled (${{ runner.os }})
+      run: |
+        python setup.py develop --uninstall
+        BUILD_MONAI=1 python setup.py develop  # compile the cpp extensions
+      shell: bash
+    - name: Run quick tests (CPU ${{ runner.os }})
+      run: |
+        python -c 'import torch; print(torch.__version__); print(torch.rand(5,3))'
+        python -c "import monai; monai.config.print_config()"
+        python -m unittest -v
+      env:
+        QUICKTEST: True
+        PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python  # https://github.com/Project-MONAI/MONAI/issues/4354
+
+  packaging:
+    runs-on: ubuntu-latest
+    env:
+      QUICKTEST: True
+      shell: bash
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+    - name: cache weekly timestamp
+      id: pip-cache
+      run: |
+        echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT
+    - name: cache for pip
+      uses: actions/cache@v4
+      id: cache
+      with:
+        path: |
+          ~/.cache/pip
+          ~/.cache/torch
+        key: ${{ runner.os }}-pip-${{ steps.pip-cache.outputs.datew }}
+    - name: Install dependencies
+      run: |
+        find  /opt/hostedtoolcache/* -maxdepth 0 ! -name 'Python' -exec rm -rf {} \;
+        python -m pip install --user --upgrade pip setuptools wheel twine packaging
+        # install the latest pytorch for testing
+        # however, "pip install monai*.tar.gz" will build cpp/cuda with an isolated
+        # fresh torch installation according to pyproject.toml
+        python -m pip install torch>=2.3.0 torchvision
+    - name: Check packages
+      run: |
+        pip uninstall monai
+        pip list | grep -iv monai
+        git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+        set -e
+
+        # build tar.gz and wheel
+        python setup.py check -m -s
+        python setup.py sdist bdist_wheel
+        python -m twine check dist/*
+    - run: echo "pwd=$PWD" >> $GITHUB_OUTPUT
+      id: root
+    - run: echo "tmp_dir=$(mktemp -d)" >> $GITHUB_OUTPUT
+      id: mktemp
+    - name: Move packages
+      run: |
+        printf ${{ steps.root.outputs.pwd }}
+        printf ${{ steps.mktemp.outputs.tmp_dir }}
+        # move packages to a temp dir
+        cp dist/monai* "${{ steps.mktemp.outputs.tmp_dir }}"
+        rm -r build dist monai.egg-info
+        cd "${{ steps.mktemp.outputs.tmp_dir }}"
+        ls -al
+    - name: Install wheel file
+      working-directory: ${{ steps.mktemp.outputs.tmp_dir }}
+      run: |
+        # install from wheel
+        python -m pip install monai*.whl
+        python -c 'import monai; monai.config.print_config()' 2>&1 | grep -iv "unknown"
+        python -c 'import monai; print(monai.__file__)'
+        python -m pip uninstall -y monai
+        rm monai*.whl
+    - name: Install source archive
+      working-directory: ${{ steps.mktemp.outputs.tmp_dir }}
+      run: |
+        # install from tar.gz
+        name=$(ls *.tar.gz | head -n1)
+        echo $name
+        python -m pip install $name[all]
+        python -c 'import monai; monai.config.print_config()' 2>&1 | grep -iv "unknown"
+        python -c 'import monai; print(monai.__file__)'
+    - name: Quick test
+      working-directory: ${{ steps.mktemp.outputs.tmp_dir }}
+      run: |
+        # run min tests
+        cp ${{ steps.root.outputs.pwd }}/requirements*.txt .
+        cp -r ${{ steps.root.outputs.pwd }}/tests .
+        ls -al
+        python -m pip install -r requirements-dev.txt
+        python -m unittest -v
+      env:
+        PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python  # https://github.com/Project-MONAI/MONAI/issues/4354
+
+  build-docs:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+    - name: cache weekly timestamp
+      id: pip-cache
+      run: |
+        echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT
+    - name: cache for pip
+      uses: actions/cache@v4
+      id: cache
+      with:
+        path: |
+          ~/.cache/pip
+          ~/.cache/torch
+        key: ${{ runner.os }}-pip-${{ steps.pip-cache.outputs.datew }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip wheel
+        python -m pip install -r docs/requirements.txt
+    - name: Make html
+      run: |
+        cd docs/
+        make clean
+        make html 2>&1 | tee tmp_log
+        if [[ $(grep -c "ERROR:" tmp_log) != 0 ]]; then echo "found errors"; grep "ERROR:" tmp_log; exit 1; fi
+        sed '/WARNING.*pip/d' tmp_log > tmp_log1; mv tmp_log1 tmp_log  # monai#7133
+        if [[ $(grep -c "WARNING:" tmp_log) != 0 ]]; then echo "found warnings"; grep "WARNING:" tmp_log; exit 1; fi
+      shell: bash

.github/workflows/weekly-preview.yml

@@ -12,7 +12,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
+<<<<<<< HEAD
+        opt: ["codeformat", "pyrefly"]
+=======
         opt: ["codeformat", "mypy"]
+>>>>>>> upstream/dev
     steps:
     - name: Clean unused tools
       run: |

.gitignore

@@ -107,15 +107,11 @@ venv.bak/
 # mkdocs documentation
 /site
 
-# pytype cache
-.pytype/
-
-# mypy
-.mypy_cache/
+# pyrefly cache
+.pyrefly_cache/
 examples/scd_lvsegs.npz
 temp/
 .idea/
-.dmypy.json
 
 *~
 

CONTRIBUTING.md

@@ -36,9 +36,14 @@ We encourage you to create pull requests early. It helps us track the contributi
 Please note that, as per PyTorch, MONAI uses American English spelling. This means classes and variables should be: normali**z**e, visuali**z**e, colo~~u~~r, etc.
 
 ### Preparing pull requests
+<<<<<<< HEAD
+To ensure the code quality, MONAI relies on several linting tools ([flake8 and its plugins](https://gitlab.com/pycqa/flake8), [black](https://github.com/psf/black), [isort](https://github.com/timothycrosley/isort), [ruff](https://github.com/astral-sh/ruff)),
+static type analysis tools ([pyrefly](https://github.com/facebook/pyrefly)), as well as a set of unit/integration tests.
+=======
 
 To ensure the code quality, MONAI relies on several linting tools ([black](https://github.com/psf/black), [isort](https://github.com/timothycrosley/isort), [ruff](https://github.com/astral-sh/ruff)),
 static type analysis tools ([mypy](https://github.com/python/mypy), [pytype](https://github.com/google/pytype)), as well as a set of unit/integration tests.
+>>>>>>> upstream/dev
 
 This section highlights all the necessary preparation steps required before sending a pull request.
 To collaborate efficiently, please read through this section and follow them.

monai/apps/auto3dseg/auto_runner.py

@@ -405,6 +405,7 @@ def inspect_datalist_folds(self, datalist_filename: str) -> int:
 
         datalist = ConfigParser.load_config_file(datalist_filename)
         if "training" not in datalist:
+            # pyrefly: ignore [unnecessary-type-conversion]
             raise ValueError("Datalist files has no training key:" + str(datalist_filename))
 
         fold_list = [int(d["fold"]) for d in datalist["training"] if "fold" in d]
@@ -790,6 +791,7 @@ def _train_algo_in_nni(self, history: list[dict[str, Any]]) -> None:
             nni_config_filename = os.path.abspath(os.path.join(self.work_dir, f"{name}_nni_config.yaml"))
             ConfigParser.export_config_file(nni_config, nni_config_filename, fmt="yaml", default_flow_style=None)
 
+            # pyrefly: ignore [redundant-cast]
             max_trial = min(self.hpo_tasks, cast(int, default_nni_config["maxTrialNumber"]))
             cmd = "nnictl create --config " + nni_config_filename + " --port 8088"
 
@@ -805,6 +807,7 @@ def _train_algo_in_nni(self, history: list[dict[str, Any]]) -> None:
                 n_trainings = len(import_bundle_algo_history(self.work_dir, only_trained=True))
 
             cmd = "nnictl stop --all"
+            # pyrefly: ignore [bad-argument-type]
             run_cmd(cmd.split(), check=True)
             logger.info(f"NNI completes HPO on {name}")
             last_total_tasks = n_trainings

monai/apps/auto3dseg/bundle_gen.py

@@ -343,6 +343,7 @@ def infer(self, image_file):
         config_dir = os.path.join(self.output_path, "configs")
         configs_path = [os.path.join(config_dir, f) for f in os.listdir(config_dir)]
 
+        # pyrefly: ignore [implicit-import]
         spec = importlib.util.spec_from_file_location("InferClass", infer_py)
         infer_class = importlib.util.module_from_spec(spec)  # type: ignore
         sys.modules["InferClass"] = infer_class

monai/apps/auto3dseg/ensemble_builder.py

@@ -337,6 +337,7 @@ def __init__(self, history: Sequence[dict[str, Any]], data_src_cfg_name: str | N
         self.ensemble: AlgoEnsemble
         self.data_src_cfg = ConfigParser(globals=False)
 
+        # pyrefly: ignore [unnecessary-type-conversion]
         if data_src_cfg_name is not None and os.path.exists(str(data_src_cfg_name)):
             self.data_src_cfg.read_config(data_src_cfg_name)
 

monai/apps/deepedit/transforms.py

@@ -434,6 +434,7 @@ def _randomize(self, d, key_label):
         else:
             logger.info(f"Not slice IDs for label: {key_label}")
             sid = None
+        # pyrefly: ignore [unsupported-operation]
         self.sid[key_label] = sid
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> dict[Hashable, np.ndarray]:
@@ -561,6 +562,7 @@ def __init__(
         self.guidance: dict[str, list[list[int]]] = {}
 
     def randomize(self, data=None):
+        # pyrefly: ignore [unsupported-operation]
         probability = data[self.probability]
         self._will_interact = self.R.choice([True, False], p=[probability, 1.0 - probability])
 
@@ -885,6 +887,7 @@ def _randomize(self, d, key_label):
         else:
             logger.info(f"Not slice IDs for label: {key_label}")
             sid = None
+        # pyrefly: ignore [unsupported-operation]
         self.sid[key_label] = sid
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> dict[Hashable, np.ndarray]:

monai/apps/deepgrow/dataset.py

@@ -175,6 +175,7 @@ def _save_data_2d(vol_idx, vol_image, vol_label, dataset_dir, relative_path):
             continue
 
         # For all Labels
+        # pyrefly: ignore [missing-attribute]
         unique_labels = np.unique(label.flatten())
         unique_labels = unique_labels[unique_labels != 0]
         unique_labels_count = max(unique_labels_count, len(unique_labels))

monai/apps/deepgrow/transforms.py

@@ -288,6 +288,7 @@ def __init__(self, guidance: str = "guidance", discrepancy: str = "discrepancy",
         self._will_interact = None
 
     def randomize(self, data=None):
+        # pyrefly: ignore [unsupported-operation]
         probability = data[self.probability]
         self._will_interact = self.R.choice([True, False], p=[probability, 1.0 - probability])
 

monai/apps/detection/networks/retinanet_detector.py

@@ -525,6 +525,7 @@ def forward(
             )
 
         # 4. Generate anchors and store it in self.anchors: List[Tensor]
+        # pyrefly: ignore [bad-argument-type]
         self.generate_anchors(images, head_outputs)
         # num_anchor_locs_per_level: List[int], list of HW or HWD for each level
         num_anchor_locs_per_level = [x.shape[2:].numel() for x in head_outputs[self.cls_key]]
@@ -535,6 +536,7 @@ def forward(
             # reshape to Tensor sized(B, sum(HWA), self.num_classes) for self.cls_key
             # or (B, sum(HWA), 2* self.spatial_dims) for self.box_reg_key
             # A = self.num_anchors_per_loc
+            # pyrefly: ignore [bad-argument-type]
             head_outputs[key] = self._reshape_maps(head_outputs[key])
 
         # 6(1). If during training, return losses