20b1 "RuntimeError: failed to find interpreter for Builtin discover of python_spec='venv1/bin/python'"

[asottile]

this is breaking some tests for pre-commit

here's a minimal case:

set -euxo pipefail
rm -rf venv1 venv2
virtualenv venv1
venv1/bin/pip install virtualenv --upgrade --pre
venv1/bin/python -mvirtualenv venv2 -p venv1/bin/python

$ bash t.sh
+ rm -rf venv1 venv2
+ virtualenv venv1
Using real prefix '/usr'
Path not in prefix '/home/asottile/opt/venv/include/python3.6m' '/usr'
New python executable in /home/asottile/workspace/pre-commit/venv1/bin/python3
Also creating executable in /home/asottile/workspace/pre-commit/venv1/bin/python
Installing setuptools, pip, wheel...
done.
+ venv1/bin/pip install virtualenv --upgrade --pre
Collecting virtualenv
  Using cached virtualenv-20.0.0b1-py2.py3-none-any.whl (4.6 MB)
Collecting six<2,>=1.12.0
  Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting importlib-metadata<2,>=0.12; python_version < "3.8"
  Using cached importlib_metadata-1.4.0-py2.py3-none-any.whl (29 kB)
Collecting appdirs<2,>=1.4.3
  Using cached appdirs-1.4.3-py2.py3-none-any.whl (12 kB)
Collecting importlib-resources<2,>=1.0; python_version < "3.7"
  Using cached importlib_resources-1.0.2-py2.py3-none-any.whl (32 kB)
Collecting filelock<4,>=3.0.0
  Using cached filelock-3.0.12-py3-none-any.whl (7.6 kB)
Collecting zipp>=0.5
  Using cached zipp-2.1.0-py3-none-any.whl (4.6 kB)
Installing collected packages: six, zipp, importlib-metadata, appdirs, importlib-resources, filelock, virtualenv
Successfully installed appdirs-1.4.3 filelock-3.0.12 importlib-metadata-1.4.0 importlib-resources-1.0.2 six-1.14.0 virtualenv-20.0.0b1 zipp-2.1.0
+ venv1/bin/python -mvirtualenv venv2 -p venv1/bin/python
Traceback (most recent call last):
  File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/asottile/workspace/pre-commit/venv1/lib/python3.6/site-packages/virtualenv/__main__.py", line 27, in <module>
    run()
  File "/home/asottile/workspace/pre-commit/venv1/lib/python3.6/site-packages/virtualenv/__main__.py", line 16, in run
    run_via_cli(args)
  File "/home/asottile/workspace/pre-commit/venv1/lib/python3.6/site-packages/virtualenv/run/__init__.py", line 21, in run_via_cli
    session = session_via_cli(args)
  File "/home/asottile/workspace/pre-commit/venv1/lib/python3.6/site-packages/virtualenv/run/__init__.py", line 33, in session_via_cli
    raise RuntimeError("failed to find interpreter for {}".format(discover))
RuntimeError: failed to find interpreter for Builtin discover of python_spec='venv1/bin/python'

$ uname -a
Linux asottile-MacBookPro 4.15.0-74-generic #84-Ubuntu SMP Thu Dec 19 08:06:28 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux
$ lsb_release -a
No LSB modules are available.
Distributor ID:	Ubuntu
Description:	Ubuntu 18.04.3 LTS
Release:	18.04
Codename:	bionic
$ venv1/bin/python --version --version
Python 3.6.8 (default, Oct  7 2019, 12:59:55) 
[GCC 8.3.0]
$ venv1/bin/pip freeze --all
appdirs==1.4.3
filelock==3.0.12
importlib-metadata==1.4.0
importlib-resources==1.0.2
pip==20.0.2
setuptools==45.1.0
six==1.14.0
virtualenv==20.0.0b1
wheel==0.34.1
zipp==2.1.0

[gaborbernat]

I think this will be #1512, cab you try rewrite branch?

[gaborbernat]

Also please post runs with -vv so we get more information 😊

[asottile]

still broken -- here's with rewrite and -vv:

set -euxo pipefail
rm -rf venv1 venv2
virtualenv venv1
venv1/bin/pip install git+https://github.com/pypa/virtualenv@rewrite
venv1/bin/python -mvirtualenv venv2 -p venv1/bin/python -vv

$ bash t.sh 
+ rm -rf venv1 venv2
+ virtualenv venv1
Using real prefix '/usr'
Path not in prefix '/home/asottile/opt/venv/include/python3.6m' '/usr'
New python executable in /home/asottile/workspace/pre-commit/venv1/bin/python3
Also creating executable in /home/asottile/workspace/pre-commit/venv1/bin/python
Installing setuptools, pip, wheel...
done.
+ venv1/bin/pip install git+https://github.com/pypa/virtualenv@rewrite
Collecting git+https://github.com/pypa/virtualenv@rewrite
  Cloning https://github.com/pypa/virtualenv (to revision rewrite) to /tmp/pip-req-build-chmb788m
  Running command git clone -q https://github.com/pypa/virtualenv /tmp/pip-req-build-chmb788m
  Running command git checkout -b rewrite --track origin/rewrite
  Switched to a new branch 'rewrite'
  Branch 'rewrite' set up to track remote branch 'rewrite' from 'origin'.
  Installing build dependencies ... done
  Getting requirements to build wheel ... done
    Preparing wheel metadata ... done
Collecting appdirs<2,>=1.4.3
  Using cached appdirs-1.4.3-py2.py3-none-any.whl (12 kB)
Collecting six<2,>=1.12.0
  Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting filelock<4,>=3.0.0
  Using cached filelock-3.0.12-py3-none-any.whl (7.6 kB)
Collecting importlib-metadata<2,>=0.12; python_version < "3.8"
  Using cached importlib_metadata-1.4.0-py2.py3-none-any.whl (29 kB)
Collecting importlib-resources<2,>=1.0; python_version < "3.7"
  Using cached importlib_resources-1.0.2-py2.py3-none-any.whl (32 kB)
Collecting zipp>=0.5
  Using cached zipp-2.1.0-py3-none-any.whl (4.6 kB)
Building wheels for collected packages: virtualenv
  Building wheel for virtualenv (PEP 517) ... done
  Created wheel for virtualenv: filename=virtualenv-0.1.dev49+g2e3f6c8-py2.py3-none-any.whl size=4594112 sha256=87830b02b5259fea8c5654cc5a1df171c6265c2645616a74bb8f877528fe562d
  Stored in directory: /tmp/pip-ephem-wheel-cache-pz32dr64/wheels/79/5c/82/576f088287bb37f13bf7bff00083b8da8570ef25e9ccd9a856
Successfully built virtualenv
Installing collected packages: appdirs, six, filelock, zipp, importlib-metadata, importlib-resources, virtualenv
Successfully installed appdirs-1.4.3 filelock-3.0.12 importlib-metadata-1.4.0 importlib-resources-1.0.2 six-1.14.0 virtualenv-0.1.dev49+g2e3f6c8 zipp-2.1.0
+ venv1/bin/python -mvirtualenv venv2 -p venv1/bin/python -vv
51 setup logging to DEBUG [DEBUG report:43]
55 find interpreter for spec PythonSpec(path=venv1/bin/python) [INFO builtin:44]
56 proposed PythonInfo(spec=CPython3.6.8.final.0-64, exe=/home/asottile/workspace/pre-commit/venv1/bin/python, platform=linux, version='3.6.8 (default, Oct  7 2019, 12:59:55) \n[GCC 8.3.0]', encoding_fs_io=utf-8-UTF-8) [INFO builtin:48]
56 discover PATH[0]=/home/asottile/bin [DEBUG builtin:75]
56 Attempting to acquire lock 140171682619120 on /home/asottile/.local/share/virtualenv/py-info/0.1.dev49+g2e3f6c8/49e501aaa4d1022cbe8557d08811d1eb0c8190d83b3a849e2b95d5d2d22b5a15.lock [DEBUG filelock:270]
56 Lock 140171682619120 acquired on /home/asottile/.local/share/virtualenv/py-info/0.1.dev49+g2e3f6c8/49e501aaa4d1022cbe8557d08811d1eb0c8190d83b3a849e2b95d5d2d22b5a15.lock [INFO filelock:274]
56 get interpreter info via cmd: /home/asottile/workspace/pre-commit/venv1/bin/python -s /home/asottile/workspace/pre-commit/venv1/lib/python3.6/site-packages/virtualenv/discovery/py_info.py [DEBUG cached_py_info:104]
114 write PythonInfo to /home/asottile/.local/share/virtualenv/py-info/0.1.dev49+g2e3f6c8/49e501aaa4d1022cbe8557d08811d1eb0c8190d83b3a849e2b95d5d2d22b5a15.json for /home/asottile/workspace/pre-commit/venv1/bin/python [DEBUG cached_py_info:85]
114 Attempting to release lock 140171682619120 on /home/asottile/.local/share/virtualenv/py-info/0.1.dev49+g2e3f6c8/49e501aaa4d1022cbe8557d08811d1eb0c8190d83b3a849e2b95d5d2d22b5a15.lock [DEBUG filelock:315]
114 Lock 140171682619120 released on /home/asottile/.local/share/virtualenv/py-info/0.1.dev49+g2e3f6c8/49e501aaa4d1022cbe8557d08811d1eb0c8190d83b3a849e2b95d5d2d22b5a15.lock [INFO filelock:318]
114 discover PATH[1]=/home/asottile/bin [DEBUG builtin:75]
114 discover PATH[2]=/usr/local/sbin [DEBUG builtin:75]
114 discover PATH[3]=/usr/local/bin [DEBUG builtin:75]
115 discover PATH[4]=/usr/sbin [DEBUG builtin:75]
115 discover PATH[5]=/usr/bin [DEBUG builtin:75]
115 Attempting to acquire lock 140171682616824 on /home/asottile/.local/share/virtualenv/py-info/0.1.dev49+g2e3f6c8/beb4f305451b4835b36a63cb1c75e625a7251cf8d76b65be2f5856eee68cb551.lock [DEBUG filelock:270]
115 Lock 140171682616824 acquired on /home/asottile/.local/share/virtualenv/py-info/0.1.dev49+g2e3f6c8/beb4f305451b4835b36a63cb1c75e625a7251cf8d76b65be2f5856eee68cb551.lock [INFO filelock:274]
115 get PythonInfo from /home/asottile/.local/share/virtualenv/py-info/0.1.dev49+g2e3f6c8/beb4f305451b4835b36a63cb1c75e625a7251cf8d76b65be2f5856eee68cb551.json for /usr/bin/python [DEBUG cached_py_info:70]
117 Attempting to release lock 140171682616824 on /home/asottile/.local/share/virtualenv/py-info/0.1.dev49+g2e3f6c8/beb4f305451b4835b36a63cb1c75e625a7251cf8d76b65be2f5856eee68cb551.lock [DEBUG filelock:315]
117 Lock 140171682616824 released on /home/asottile/.local/share/virtualenv/py-info/0.1.dev49+g2e3f6c8/beb4f305451b4835b36a63cb1c75e625a7251cf8d76b65be2f5856eee68cb551.lock [INFO filelock:318]
117 proposed PythonInfo(spec=CPython2.7.17.final.0-64, exe=/usr/bin/python, platform=linux2, version='2.7.17 (default, Nov  7 2019, 10:07:09) \n[GCC 7.4.0]', encoding_fs_io=UTF-8-None) [INFO builtin:48]
117 discover PATH[6]=/sbin [DEBUG builtin:75]
117 discover PATH[7]=/bin [DEBUG builtin:75]
118 discover PATH[8]=/usr/games [DEBUG builtin:75]
118 discover PATH[9]=/usr/local/games [DEBUG builtin:75]
118 done in 67ms [INFO __main__:23]
Traceback (most recent call last):
  File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/asottile/workspace/pre-commit/venv1/lib/python3.6/site-packages/virtualenv/__main__.py", line 27, in <module>
    run()
  File "/home/asottile/workspace/pre-commit/venv1/lib/python3.6/site-packages/virtualenv/__main__.py", line 16, in run
    run_via_cli(args)
  File "/home/asottile/workspace/pre-commit/venv1/lib/python3.6/site-packages/virtualenv/run/__init__.py", line 21, in run_via_cli
    session = session_via_cli(args)
  File "/home/asottile/workspace/pre-commit/venv1/lib/python3.6/site-packages/virtualenv/run/__init__.py", line 33, in session_via_cli
    raise RuntimeError("failed to find interpreter for {}".format(discover))
RuntimeError: failed to find interpreter for Builtin discover of python_spec='venv1/bin/python'

[gaborbernat]

This seems like a valid bug, we probably don't handle relative paths yet, will take a look 👍

[asottile]

it may also be helpful to note that venv1/bin/python is itself a symlink:

$ ls -al venv1/bin/python
lrwxrwxrwx 1 asottile asottile 7 Jan 28 16:04 venv1/bin/python -> python3

just noticed this while looking at it

[gaborbernat]

Fixed in #1523

[asottile]

still failing for me, but now the same way as in #1515 so I'll continue that discussion there:

$ bash t.sh 
+ rm -rf venv1 venv2
+ virtualenv venv1
Using real prefix '/usr'
Path not in prefix '/home/asottile/opt/venv/include/python3.6m' '/usr'
New python executable in /tmp/x/venv1/bin/python3
Also creating executable in /tmp/x/venv1/bin/python
Installing setuptools, pip, wheel...
done.
+ venv1/bin/pip install git+https://github.com/pypa/virtualenv@rewrite
Collecting git+https://github.com/pypa/virtualenv@rewrite
  Cloning https://github.com/pypa/virtualenv (to revision rewrite) to /tmp/pip-req-build-elbdtx18
  Running command git clone -q https://github.com/pypa/virtualenv /tmp/pip-req-build-elbdtx18
  Running command git checkout -b rewrite --track origin/rewrite
  Switched to a new branch 'rewrite'
  Branch 'rewrite' set up to track remote branch 'rewrite' from 'origin'.
  Installing build dependencies ... done
  Getting requirements to build wheel ... done
    Preparing wheel metadata ... done
Collecting six<2,>=1.12.0
  Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting importlib-metadata<2,>=0.12; python_version < "3.8"
  Using cached importlib_metadata-1.5.0-py2.py3-none-any.whl (30 kB)
Collecting importlib-resources<2,>=1.0; python_version < "3.7"
  Using cached importlib_resources-1.0.2-py2.py3-none-any.whl (32 kB)
Collecting filelock<4,>=3.0.0
  Using cached filelock-3.0.12-py3-none-any.whl (7.6 kB)
Collecting appdirs<2,>=1.4.3
  Using cached appdirs-1.4.3-py2.py3-none-any.whl (12 kB)
Collecting zipp>=0.5
  Using cached zipp-2.1.0-py3-none-any.whl (4.6 kB)
Building wheels for collected packages: virtualenv
  Building wheel for virtualenv (PEP 517) ... done
  Created wheel for virtualenv: filename=virtualenv-20.0.0b2.dev7+g5629b41-py2.py3-none-any.whl size=4594700 sha256=560cb60f6d4b2f26c12424d877974b5da98e4df5497f318cb0f3319e42adf5cc
  Stored in directory: /tmp/pip-ephem-wheel-cache-_j7v5ygw/wheels/79/5c/82/576f088287bb37f13bf7bff00083b8da8570ef25e9ccd9a856
Successfully built virtualenv
Installing collected packages: six, zipp, importlib-metadata, importlib-resources, filelock, appdirs, virtualenv
Successfully installed appdirs-1.4.3 filelock-3.0.12 importlib-metadata-1.5.0 importlib-resources-1.0.2 six-1.14.0 virtualenv-20.0.0b2.dev7+g5629b41 zipp-2.1.0
+ venv1/bin/python -mvirtualenv venv2 -p venv1/bin/python -vv
49 setup logging to DEBUG [DEBUG report:43]
53 find interpreter for spec PythonSpec(path=/tmp/x/venv1/bin/python) [INFO builtin:44]
53 proposed PythonInfo(spec=CPython3.6.8.final.0-64, exe=/tmp/x/venv1/bin/python, platform=linux, version='3.6.8 (default, Oct  7 2019, 12:59:55) \n[GCC 8.3.0]', encoding_fs_io=utf-8-UTF-8) [INFO builtin:48]
53 accepted target interpreter PythonInfo(spec=CPython3.6.8.final.0-64, exe=/tmp/x/venv1/bin/python, platform=linux, version='3.6.8 (default, Oct  7 2019, 12:59:55) \n[GCC 8.3.0]', encoding_fs_io=utf-8-UTF-8) [DEBUG builtin:50]
54 filesystem is case-sensitive [DEBUG info:28]
59 Attempting to acquire lock 139944776433616 on /home/asottile/.local/share/virtualenv/py-info/20.0.0b2.dev7+g5629b41/30295c5bec572e859485b1ffa5e89b8b3e2022ef6e3e739c1ac40f143a557caf.lock [DEBUG filelock:270]
59 Lock 139944776433616 acquired on /home/asottile/.local/share/virtualenv/py-info/20.0.0b2.dev7+g5629b41/30295c5bec572e859485b1ffa5e89b8b3e2022ef6e3e739c1ac40f143a557caf.lock [INFO filelock:274]
59 get PythonInfo from /home/asottile/.local/share/virtualenv/py-info/20.0.0b2.dev7+g5629b41/30295c5bec572e859485b1ffa5e89b8b3e2022ef6e3e739c1ac40f143a557caf.json for /usr/bin/python3.6 [DEBUG cached_py_info:70]
60 Attempting to release lock 139944776433616 on /home/asottile/.local/share/virtualenv/py-info/20.0.0b2.dev7+g5629b41/30295c5bec572e859485b1ffa5e89b8b3e2022ef6e3e739c1ac40f143a557caf.lock [DEBUG filelock:315]
60 Lock 139944776433616 released on /home/asottile/.local/share/virtualenv/py-info/20.0.0b2.dev7+g5629b41/30295c5bec572e859485b1ffa5e89b8b3e2022ef6e3e739c1ac40f143a557caf.lock [INFO filelock:318]
60 Attempting to acquire lock 139944776432944 on /home/asottile/.local/share/virtualenv/py-info/20.0.0b2.dev7+g5629b41/beb4f305451b4835b36a63cb1c75e625a7251cf8d76b65be2f5856eee68cb551.lock [DEBUG filelock:270]
60 Lock 139944776432944 acquired on /home/asottile/.local/share/virtualenv/py-info/20.0.0b2.dev7+g5629b41/beb4f305451b4835b36a63cb1c75e625a7251cf8d76b65be2f5856eee68cb551.lock [INFO filelock:274]
60 get interpreter info via cmd: /usr/bin/python -s /tmp/x/venv1/lib/python3.6/site-packages/virtualenv/discovery/py_info.py [DEBUG cached_py_info:104]
84 write PythonInfo to /home/asottile/.local/share/virtualenv/py-info/20.0.0b2.dev7+g5629b41/beb4f305451b4835b36a63cb1c75e625a7251cf8d76b65be2f5856eee68cb551.json for /usr/bin/python [DEBUG cached_py_info:85]
84 Attempting to release lock 139944776432944 on /home/asottile/.local/share/virtualenv/py-info/20.0.0b2.dev7+g5629b41/beb4f305451b4835b36a63cb1c75e625a7251cf8d76b65be2f5856eee68cb551.lock [DEBUG filelock:315]
84 Lock 139944776432944 released on /home/asottile/.local/share/virtualenv/py-info/20.0.0b2.dev7+g5629b41/beb4f305451b4835b36a63cb1c75e625a7251cf8d76b65be2f5856eee68cb551.lock [INFO filelock:318]
84 done in 36ms [INFO __main__:23]
Traceback (most recent call last):
  File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/tmp/x/venv1/lib/python3.6/site-packages/virtualenv/__main__.py", line 27, in <module>
    run()
  File "/tmp/x/venv1/lib/python3.6/site-packages/virtualenv/__main__.py", line 16, in run
    run_via_cli(args)
  File "/tmp/x/venv1/lib/python3.6/site-packages/virtualenv/run/__init__.py", line 21, in run_via_cli
    session = session_via_cli(args)
  File "/tmp/x/venv1/lib/python3.6/site-packages/virtualenv/run/__init__.py", line 35, in session_via_cli
    CreatorSelector(interpreter, parser),
  File "/tmp/x/venv1/lib/python3.6/site-packages/virtualenv/run/plugin/creators.py", line 15, in __init__
    creators, self.key_to_meta, self.describe, self.builtin_key = self.for_interpreter(interpreter)
  File "/tmp/x/venv1/lib/python3.6/site-packages/virtualenv/run/plugin/creators.py", line 26, in for_interpreter
    meta = creator_class.can_create(interpreter)
  File "/tmp/x/venv1/lib/python3.6/site-packages/virtualenv/create/via_global_ref/builtin/via_global_self_do.py", line 33, in can_create
    for src in cls.sources(interpreter):
  File "/tmp/x/venv1/lib/python3.6/site-packages/virtualenv/create/via_global_ref/builtin/via_global_self_do.py", line 54, in sources
    for host_exe, targets in cls._executables(interpreter):
  File "/tmp/x/venv1/lib/python3.6/site-packages/virtualenv/create/via_global_ref/builtin/cpython/common.py", line 31, in _executables
    host_exe = Path(interpreter.system_executable)
  File "/tmp/x/venv1/lib/python3.6/site-packages/virtualenv/discovery/py_info.py", line 203, in system_executable
    result = self.find_exe_based_of(inside_folder=env_prefix)
  File "/tmp/x/venv1/lib/python3.6/site-packages/virtualenv/discovery/py_info.py", line 225, in find_exe_based_of
    "failed to detect {} in {}".format(what, os.pathsep.join(possible_folders))
RuntimeError: failed to detect cpython3.6.8-64|cpython3.6.8|cpython3.6-64|cpython3.6|cpython3-64|cpython3|cpython-64|cpython|CPython3.6.8-64|CPython3.6.8|CPython3.6-64|CPython3.6|CPython3-64|CPython3|CPython-64|CPython|CPYTHON3.6.8-64|CPYTHON3.6.8|CPYTHON3.6-64|CPYTHON3.6|CPYTHON3-64|CPYTHON3|CPYTHON-64|CPYTHON|python3.6.8-64|python3.6.8|python3.6-64|python3.6|python3-64|python3|python-64|python|PYTHON3.6.8-64|PYTHON3.6.8|PYTHON3.6-64|PYTHON3.6|PYTHON3-64|PYTHON3|PYTHON-64|PYTHON in /usr/bin:/usr