Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[READY] Only add the necessary directories to Python path #1124

Merged
merged 1 commit into from
Oct 18, 2018
Merged

[READY] Only add the necessary directories to Python path #1124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 26 additions & 28 deletions .ycm_extra_conf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,12 @@

from distutils.sysconfig import get_python_inc
import platform
import os
import os.path as p
import subprocess
import ycm_core

DIR_OF_THIS_SCRIPT = os.path.abspath( os.path.dirname( __file__ ) )
DIR_OF_THIRD_PARTY = os.path.join( DIR_OF_THIS_SCRIPT, 'third_party' )
DIR_OF_THIS_SCRIPT = p.abspath( p.dirname( __file__ ) )
DIR_OF_THIRD_PARTY = p.join( DIR_OF_THIS_SCRIPT, 'third_party' )
SOURCE_EXTENSIONS = [ '.cpp', '.cxx', '.cc', '.c', '.m', '.mm' ]

# These are the compilation flags that will be used in case there's no
Expand Down Expand Up @@ -104,30 +104,30 @@
# 'flags' list of compilation flags. Notice that YCM itself uses that approach.
compilation_database_folder = ''

if os.path.exists( compilation_database_folder ):
if p.exists( compilation_database_folder ):
database = ycm_core.CompilationDatabase( compilation_database_folder )
else:
database = None


def IsHeaderFile( filename ):
extension = os.path.splitext( filename )[ 1 ]
extension = p.splitext( filename )[ 1 ]
return extension in [ '.h', '.hxx', '.hpp', '.hh' ]


def FindCorrespondingSourceFile( filename ):
if IsHeaderFile( filename ):
basename = os.path.splitext( filename )[ 0 ]
basename = p.splitext( filename )[ 0 ]
for extension in SOURCE_EXTENSIONS:
replacement_file = basename + extension
if os.path.exists( replacement_file ):
if p.exists( replacement_file ):
return replacement_file
return filename


def PathToPythonUsedDuringBuild():
try:
filepath = os.path.join( DIR_OF_THIS_SCRIPT, 'PYTHON_USED_DURING_BUILDING' )
filepath = p.join( DIR_OF_THIS_SCRIPT, 'PYTHON_USED_DURING_BUILDING' )
with open( filepath ) as f:
return f.read().strip()
# We need to check for IOError for Python 2 and OSError for Python 3.
Expand Down Expand Up @@ -186,31 +186,29 @@ def Settings( **kwargs ):

def GetStandardLibraryIndexInSysPath( sys_path ):
for index, path in enumerate( sys_path ):
if os.path.isfile( os.path.join( path, 'os.py' ) ):
if p.isfile( p.join( path, 'os.py' ) ):
return index
raise RuntimeError( 'Could not find standard library path in Python path.' )


def PythonSysPath( **kwargs ):
sys_path = kwargs[ 'sys_path' ]

sys_path.insert( 0, DIR_OF_THIS_SCRIPT )

for folder in os.listdir( DIR_OF_THIRD_PARTY ):
if folder == 'python-future':
folder = os.path.join( folder, 'src' )
sys_path.insert( GetStandardLibraryIndexInSysPath( sys_path ) + 1,
os.path.realpath( os.path.join( DIR_OF_THIRD_PARTY,
folder ) ) )
continue

if folder == 'cregex':
interpreter_path = kwargs[ 'interpreter_path' ]
major_version = subprocess.check_output( [
interpreter_path, '-c', 'import sys; print( sys.version_info[ 0 ] )' ]
).rstrip().decode( 'utf8' )
folder = os.path.join( folder, 'regex_{}'.format( major_version ) )

sys_path.insert( 0, os.path.realpath( os.path.join( DIR_OF_THIRD_PARTY,
folder ) ) )
interpreter_path = kwargs[ 'interpreter_path' ]
major_version = subprocess.check_output( [
interpreter_path, '-c', 'import sys; print( sys.version_info[ 0 ] )' ]
).rstrip().decode( 'utf8' )

sys_path.insert( GetStandardLibraryIndexInSysPath( sys_path ) + 1,
p.join( DIR_OF_THIRD_PARTY, 'python-future', 'src' ) )
sys_path[ 0:0 ] = [ p.join( DIR_OF_THIS_SCRIPT ),
p.join( DIR_OF_THIRD_PARTY, 'bottle' ),
p.join( DIR_OF_THIRD_PARTY, 'cregex',
'regex_{}'.format( major_version ) ),
p.join( DIR_OF_THIRD_PARTY, 'frozendict' ),
p.join( DIR_OF_THIRD_PARTY, 'jedi' ),
p.join( DIR_OF_THIRD_PARTY, 'parso' ),
p.join( DIR_OF_THIRD_PARTY, 'requests' ),
p.join( DIR_OF_THIRD_PARTY, 'waitress' ) ]

return sys_path
103 changes: 30 additions & 73 deletions ycmd/server_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@

import io
import logging
import os
import os.path as p
import re
import sys

Expand Down Expand Up @@ -70,14 +70,15 @@

VERSION_FILENAME = 'CORE_VERSION'

DIR_OF_CURRENT_SCRIPT = os.path.dirname( os.path.abspath( __file__ ) )
ROOT_DIR = p.normpath( p.join( p.dirname( __file__ ), '..' ) )
DIR_OF_THIRD_PARTY = p.join( ROOT_DIR, 'third_party' )
DIR_PACKAGES_REGEX = re.compile( '(site|dist)-packages$' )

_logger = logging.getLogger( __name__ )


def ExpectedCoreVersion():
filepath = os.path.join( DIR_OF_CURRENT_SCRIPT, '..', VERSION_FILENAME )
filepath = p.join( ROOT_DIR, VERSION_FILENAME )
with io.open( filepath, encoding = 'utf8' ) as f:
return int( f.read() )

Expand Down Expand Up @@ -121,48 +122,14 @@ def CompatibleWithCurrentCore():
return CORE_COMPATIBLE_STATUS


def SetUpPythonPath():
sys.path.insert( 0, os.path.join( DIR_OF_CURRENT_SCRIPT, '..' ) )

# We don't add this path in AddNearestThirdPartyFoldersToSysPath because
# loading the regex module in YCM may cause a segmentation fault if the module
# is compiled for a different version of Python than the one running YCM.
regex_folder = os.path.join( DIR_OF_CURRENT_SCRIPT,
'..',
'third_party',
'cregex',
'regex_{}'.format( sys.version_info[ 0 ] ) )
sys.path.insert( 0, regex_folder )

AddNearestThirdPartyFoldersToSysPath( __file__ )


def AncestorFolders( path ):
folder = os.path.normpath( path )
while True:
parent = os.path.dirname( folder )
if parent == folder:
break
folder = parent
yield folder


def PathToNearestThirdPartyFolder( path ):
for folder in AncestorFolders( path ):
path_to_third_party = os.path.join( folder, 'third_party' )
if os.path.isdir( path_to_third_party ):
return path_to_third_party
return None


def IsStandardLibraryFolder( path ):
return ( ( os.path.isfile( path )
and PYTHON_STDLIB_ZIP_REGEX.match( os.path.basename( path ) ) )
or os.path.isfile( os.path.join( path, 'os.py' ) ) )
return ( ( p.isfile( path )
and PYTHON_STDLIB_ZIP_REGEX.match( p.basename( path ) ) )
or p.isfile( p.join( path, 'os.py' ) ) )


def IsVirtualEnvLibraryFolder( path ):
return os.path.isfile( os.path.join( path, 'orig-prefix.txt' ) )
return p.isfile( p.join( path, 'orig-prefix.txt' ) )


def GetStandardLibraryIndexInSysPath():
Expand All @@ -173,35 +140,25 @@ def GetStandardLibraryIndexInSysPath():
raise RuntimeError( 'Could not find standard library path in Python path.' )


def AddNearestThirdPartyFoldersToSysPath( filepath ):
path_to_third_party = PathToNearestThirdPartyFolder( filepath )
if not path_to_third_party:
raise RuntimeError(
'No third_party folder found for: {0}'.format( filepath ) )

# NOTE: Any hacks for loading modules that can't be imported without custom
# logic need to be reproduced in run_tests.py as well.
for folder in os.listdir( path_to_third_party ):
# python-future needs special handling. Not only does it store the modules
# under its 'src' folder, but SOME of its modules are only meant to be
# accessible under py2, not py3. This is because these modules (like
# `queue`) are implementations of modules present in the py3 standard
# library. Furthermore, we need to be sure that they are not overridden by
# already installed packages (for example, the 'builtins' module from
# 'pies2overrides' or a different version of 'python-future'). To work
# around these issues, we place the python-future just after the Python
# standard library so that its modules can be overridden by standard
# modules but not by installed packages.
if folder == 'python-future':
folder = os.path.join( folder, 'src' )
sys.path.insert( GetStandardLibraryIndexInSysPath() + 1,
os.path.realpath( os.path.join( path_to_third_party,
folder ) ) )
continue

# The regex module is already included in SetUpPythonPath.
if folder == 'cregex':
continue

sys.path.insert( 0, os.path.realpath( os.path.join( path_to_third_party,
folder ) ) )
def SetUpPythonPath():
# python-future needs special handling. Not only does it store the modules
# under its 'src' folder, but SOME of its modules are only meant to be
# accessible under py2, not py3. This is because these modules (like
# `queue`) are implementations of modules present in the py3 standard
# library. Furthermore, we need to be sure that they are not overridden by
# already installed packages (for example, the 'builtins' module from
# 'pies2overrides' or a different version of 'python-future'). To work
# around these issues, we place the python-future just after the Python
# standard library so that its modules can be overridden by standard
# modules but not by installed packages.
sys.path.insert( GetStandardLibraryIndexInSysPath() + 1,
p.join( DIR_OF_THIRD_PARTY, 'python-future', 'src' ) )
sys.path[ 0:0 ] = [ p.join( ROOT_DIR ),
p.join( DIR_OF_THIRD_PARTY, 'bottle' ),
p.join( DIR_OF_THIRD_PARTY, 'cregex',
'regex_{}'.format( sys.version_info[ 0 ] ) ),
p.join( DIR_OF_THIRD_PARTY, 'frozendict' ),
p.join( DIR_OF_THIRD_PARTY, 'jedi' ),
p.join( DIR_OF_THIRD_PARTY, 'parso' ),
p.join( DIR_OF_THIRD_PARTY, 'requests' ),
p.join( DIR_OF_THIRD_PARTY, 'waitress' ) ]
95 changes: 4 additions & 91 deletions ycmd/tests/server_utils_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,35 +22,12 @@
# Not installing aliases from python-future; it's unreliable and slow.
from builtins import * # noqa

from hamcrest import ( assert_that, calling, contains, contains_inanyorder,
empty, equal_to, has_length, raises )
from hamcrest import assert_that, calling, empty, equal_to, has_length, raises
from mock import patch
from nose.tools import ok_
import os.path
import sys

from ycmd.server_utils import ( AddNearestThirdPartyFoldersToSysPath,
CompatibleWithCurrentCore,
GetStandardLibraryIndexInSysPath,
PathToNearestThirdPartyFolder )
from ycmd.tests import PathToTestFile

DIR_OF_THIRD_PARTY = os.path.abspath(
os.path.join( os.path.dirname( __file__ ), '..', '..', 'third_party' ) )
THIRD_PARTY_FOLDERS = [
os.path.join( DIR_OF_THIRD_PARTY, 'bottle' ),
os.path.join( DIR_OF_THIRD_PARTY, 'frozendict' ),
os.path.join( DIR_OF_THIRD_PARTY, 'go' ),
os.path.join( DIR_OF_THIRD_PARTY, 'jedi' ),
os.path.join( DIR_OF_THIRD_PARTY, 'OmniSharpServer' ),
os.path.join( DIR_OF_THIRD_PARTY, 'parso' ),
os.path.join( DIR_OF_THIRD_PARTY, 'racerd' ),
os.path.join( DIR_OF_THIRD_PARTY, 'requests' ),
os.path.join( DIR_OF_THIRD_PARTY, 'tern_runtime' ),
os.path.join( DIR_OF_THIRD_PARTY, 'tsserver' ),
os.path.join( DIR_OF_THIRD_PARTY, 'waitress' ),
os.path.join( DIR_OF_THIRD_PARTY, 'eclipse.jdt.ls' ),
]
from ycmd.server_utils import ( CompatibleWithCurrentCore,
GetStandardLibraryIndexInSysPath )
from ycmd.tests import PathToTestFile


@patch( 'ycmd.server_utils._logger', autospec = True )
Expand Down Expand Up @@ -155,70 +132,6 @@ def CompatibleWithCurrentCore_Outdated_NoVersionMatch_test( logger, *args ):
'script. See the documentation for more details.' )


def PathToNearestThirdPartyFolder_Success_test():
ok_( PathToNearestThirdPartyFolder( os.path.abspath( __file__ ) ) )


def PathToNearestThirdPartyFolder_Failure_test():
ok_( not PathToNearestThirdPartyFolder( os.path.expanduser( '~' ) ) )


def AddNearestThirdPartyFoldersToSysPath_Failure_test():
assert_that(
calling( AddNearestThirdPartyFoldersToSysPath ).with_args(
os.path.expanduser( '~' ) ),
raises( RuntimeError, '.*third_party folder.*' ) )


@patch( 'sys.path', [
PathToTestFile( 'python-future', 'some', 'path' ),
PathToTestFile( 'python-future', 'standard_library' ),
PathToTestFile( 'python-future', 'standard_library', 'site-packages' ),
PathToTestFile( 'python-future', 'another', 'path' ) ] )
def AddNearestThirdPartyFoldersToSysPath_FutureAfterStandardLibrary_test(
*args ):
AddNearestThirdPartyFoldersToSysPath( __file__ )
assert_that( sys.path[ : len( THIRD_PARTY_FOLDERS ) ], contains_inanyorder(
*THIRD_PARTY_FOLDERS
) )
assert_that( sys.path[ len( THIRD_PARTY_FOLDERS ) : ], contains(
PathToTestFile( 'python-future', 'some', 'path' ),
PathToTestFile( 'python-future', 'standard_library' ),
os.path.join( DIR_OF_THIRD_PARTY, 'python-future', 'src' ),
PathToTestFile( 'python-future', 'standard_library', 'site-packages' ),
PathToTestFile( 'python-future', 'another', 'path' )
) )


@patch( 'sys.path', [
PathToTestFile( 'python-future', 'some', 'path' ),
PathToTestFile( 'python-future', 'another', 'path' ) ] )
def AddNearestThirdPartyFoldersToSysPath_ErrorIfNoStandardLibrary_test( *args ):
assert_that(
calling( AddNearestThirdPartyFoldersToSysPath ).with_args( __file__ ),
raises( RuntimeError,
'Could not find standard library path in Python path.' ) )


@patch( 'sys.path', [
PathToTestFile( 'python-future', 'some', 'path' ),
PathToTestFile( 'python-future', 'virtualenv_library' ),
PathToTestFile( 'python-future', 'standard_library' ),
PathToTestFile( 'python-future', 'another', 'path' ) ] )
def AddNearestThirdPartyFoldersToSysPath_IgnoreVirtualEnvLibrary_test( *args ):
AddNearestThirdPartyFoldersToSysPath( __file__ )
assert_that( sys.path[ : len( THIRD_PARTY_FOLDERS ) ], contains_inanyorder(
*THIRD_PARTY_FOLDERS
) )
assert_that( sys.path[ len( THIRD_PARTY_FOLDERS ) : ], contains(
PathToTestFile( 'python-future', 'some', 'path' ),
PathToTestFile( 'python-future', 'virtualenv_library' ),
PathToTestFile( 'python-future', 'standard_library' ),
os.path.join( DIR_OF_THIRD_PARTY, 'python-future', 'src' ),
PathToTestFile( 'python-future', 'another', 'path' )
) )


@patch( 'sys.path', [
PathToTestFile( 'python-future', 'some', 'path' ),
PathToTestFile( 'python-future', 'another', 'path' ) ] )
Expand Down