Auto merge of #1124 - micbou:python-path, r=bstaletic

[READY] Only add the necessary directories to Python path

See PR ycm-core/YouCompleteMe#3163.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/valloric/ycmd/1124)
<!-- Reviewable:end -->

.ycm_extra_conf.py

@@ -30,12 +30,12 @@
 
 from distutils.sysconfig import get_python_inc
 import platform
-import os
+import os.path as p
 import subprocess
 import ycm_core
 
-DIR_OF_THIS_SCRIPT = os.path.abspath( os.path.dirname( __file__ ) )
-DIR_OF_THIRD_PARTY = os.path.join( DIR_OF_THIS_SCRIPT, 'third_party' )
+DIR_OF_THIS_SCRIPT = p.abspath( p.dirname( __file__ ) )
+DIR_OF_THIRD_PARTY = p.join( DIR_OF_THIS_SCRIPT, 'third_party' )
 SOURCE_EXTENSIONS = [ '.cpp', '.cxx', '.cc', '.c', '.m', '.mm' ]
 
 # These are the compilation flags that will be used in case there's no
@@ -104,30 +104,30 @@
 # 'flags' list of compilation flags. Notice that YCM itself uses that approach.
 compilation_database_folder = ''
 
-if os.path.exists( compilation_database_folder ):
+if p.exists( compilation_database_folder ):
   database = ycm_core.CompilationDatabase( compilation_database_folder )
 else:
   database = None
 
 
 def IsHeaderFile( filename ):
-  extension = os.path.splitext( filename )[ 1 ]
+  extension = p.splitext( filename )[ 1 ]
   return extension in [ '.h', '.hxx', '.hpp', '.hh' ]
 
 
 def FindCorrespondingSourceFile( filename ):
   if IsHeaderFile( filename ):
-    basename = os.path.splitext( filename )[ 0 ]
+    basename = p.splitext( filename )[ 0 ]
     for extension in SOURCE_EXTENSIONS:
       replacement_file = basename + extension
-      if os.path.exists( replacement_file ):
+      if p.exists( replacement_file ):
         return replacement_file
   return filename
 
 
 def PathToPythonUsedDuringBuild():
   try:
-    filepath = os.path.join( DIR_OF_THIS_SCRIPT, 'PYTHON_USED_DURING_BUILDING' )
+    filepath = p.join( DIR_OF_THIS_SCRIPT, 'PYTHON_USED_DURING_BUILDING' )
     with open( filepath ) as f:
       return f.read().strip()
   # We need to check for IOError for Python 2 and OSError for Python 3.
@@ -186,31 +186,29 @@ def Settings( **kwargs ):
 
 def GetStandardLibraryIndexInSysPath( sys_path ):
   for index, path in enumerate( sys_path ):
-    if os.path.isfile( os.path.join( path, 'os.py' ) ):
+    if p.isfile( p.join( path, 'os.py' ) ):
       return index
   raise RuntimeError( 'Could not find standard library path in Python path.' )
 
 
 def PythonSysPath( **kwargs ):
   sys_path = kwargs[ 'sys_path' ]
 
-  sys_path.insert( 0, DIR_OF_THIS_SCRIPT )
-
-  for folder in os.listdir( DIR_OF_THIRD_PARTY ):
-    if folder == 'python-future':
-      folder = os.path.join( folder, 'src' )
-      sys_path.insert( GetStandardLibraryIndexInSysPath( sys_path ) + 1,
-                       os.path.realpath( os.path.join( DIR_OF_THIRD_PARTY,
-                                                       folder ) ) )
-      continue
-
-    if folder == 'cregex':
-      interpreter_path = kwargs[ 'interpreter_path' ]
-      major_version = subprocess.check_output( [
-        interpreter_path, '-c', 'import sys; print( sys.version_info[ 0 ] )' ]
-      ).rstrip().decode( 'utf8' )
-      folder = os.path.join( folder, 'regex_{}'.format( major_version ) )
-
-    sys_path.insert( 0, os.path.realpath( os.path.join( DIR_OF_THIRD_PARTY,
-                                                        folder ) ) )
+  interpreter_path = kwargs[ 'interpreter_path' ]
+  major_version = subprocess.check_output( [
+    interpreter_path, '-c', 'import sys; print( sys.version_info[ 0 ] )' ]
+  ).rstrip().decode( 'utf8' )
+
+  sys_path.insert( GetStandardLibraryIndexInSysPath( sys_path ) + 1,
+                   p.join( DIR_OF_THIRD_PARTY, 'python-future', 'src' ) )
+  sys_path[ 0:0 ] = [ p.join( DIR_OF_THIS_SCRIPT ),
+                      p.join( DIR_OF_THIRD_PARTY, 'bottle' ),
+                      p.join( DIR_OF_THIRD_PARTY, 'cregex',
+                              'regex_{}'.format( major_version ) ),
+                      p.join( DIR_OF_THIRD_PARTY, 'frozendict' ),
+                      p.join( DIR_OF_THIRD_PARTY, 'jedi' ),
+                      p.join( DIR_OF_THIRD_PARTY, 'parso' ),
+                      p.join( DIR_OF_THIRD_PARTY, 'requests' ),
+                      p.join( DIR_OF_THIRD_PARTY, 'waitress' ) ]
+
   return sys_path

ycmd/server_utils.py

@@ -24,7 +24,7 @@
 
 import io
 import logging
-import os
+import os.path as p
 import re
 import sys
 
@@ -70,14 +70,15 @@
 
 VERSION_FILENAME = 'CORE_VERSION'
 
-DIR_OF_CURRENT_SCRIPT = os.path.dirname( os.path.abspath( __file__ ) )
+ROOT_DIR = p.normpath( p.join( p.dirname( __file__ ), '..' ) )
+DIR_OF_THIRD_PARTY = p.join( ROOT_DIR, 'third_party' )
 DIR_PACKAGES_REGEX = re.compile( '(site|dist)-packages$' )
 
 _logger = logging.getLogger( __name__ )
 
 
 def ExpectedCoreVersion():
-  filepath = os.path.join( DIR_OF_CURRENT_SCRIPT, '..', VERSION_FILENAME )
+  filepath = p.join( ROOT_DIR, VERSION_FILENAME )
   with io.open( filepath, encoding = 'utf8' ) as f:
     return int( f.read() )
 
@@ -121,48 +122,14 @@ def CompatibleWithCurrentCore():
   return CORE_COMPATIBLE_STATUS
 
 
-def SetUpPythonPath():
-  sys.path.insert( 0, os.path.join( DIR_OF_CURRENT_SCRIPT, '..' ) )
-
-  # We don't add this path in AddNearestThirdPartyFoldersToSysPath because
-  # loading the regex module in YCM may cause a segmentation fault if the module
-  # is compiled for a different version of Python than the one running YCM.
-  regex_folder = os.path.join( DIR_OF_CURRENT_SCRIPT,
-                               '..',
-                               'third_party',
-                               'cregex',
-                               'regex_{}'.format( sys.version_info[ 0 ] ) )
-  sys.path.insert( 0, regex_folder )
-
-  AddNearestThirdPartyFoldersToSysPath( __file__ )
-
-
-def AncestorFolders( path ):
-  folder = os.path.normpath( path )
-  while True:
-    parent = os.path.dirname( folder )
-    if parent == folder:
-      break
-    folder = parent
-    yield folder
-
-
-def PathToNearestThirdPartyFolder( path ):
-  for folder in AncestorFolders( path ):
-    path_to_third_party = os.path.join( folder, 'third_party' )
-    if os.path.isdir( path_to_third_party ):
-      return path_to_third_party
-  return None
-
-
 def IsStandardLibraryFolder( path ):
-  return ( ( os.path.isfile( path )
-             and PYTHON_STDLIB_ZIP_REGEX.match( os.path.basename( path ) ) )
-           or os.path.isfile( os.path.join( path, 'os.py' ) ) )
+  return ( ( p.isfile( path )
+             and PYTHON_STDLIB_ZIP_REGEX.match( p.basename( path ) ) )
+           or p.isfile( p.join( path, 'os.py' ) ) )
 
 
 def IsVirtualEnvLibraryFolder( path ):
-  return os.path.isfile( os.path.join( path, 'orig-prefix.txt' ) )
+  return p.isfile( p.join( path, 'orig-prefix.txt' ) )
 
 
 def GetStandardLibraryIndexInSysPath():
@@ -173,35 +140,25 @@ def GetStandardLibraryIndexInSysPath():
   raise RuntimeError( 'Could not find standard library path in Python path.' )
 
 
-def AddNearestThirdPartyFoldersToSysPath( filepath ):
-  path_to_third_party = PathToNearestThirdPartyFolder( filepath )
-  if not path_to_third_party:
-    raise RuntimeError(
-        'No third_party folder found for: {0}'.format( filepath ) )
-
-  # NOTE: Any hacks for loading modules that can't be imported without custom
-  # logic need to be reproduced in run_tests.py as well.
-  for folder in os.listdir( path_to_third_party ):
-    # python-future needs special handling. Not only does it store the modules
-    # under its 'src' folder, but SOME of its modules are only meant to be
-    # accessible under py2, not py3. This is because these modules (like
-    # `queue`) are implementations of modules present in the py3 standard
-    # library. Furthermore, we need to be sure that they are not overridden by
-    # already installed packages (for example, the 'builtins' module from
-    # 'pies2overrides' or a different version of 'python-future'). To work
-    # around these issues, we place the python-future just after the Python
-    # standard library so that its modules can be overridden by standard
-    # modules but not by installed packages.
-    if folder == 'python-future':
-      folder = os.path.join( folder, 'src' )
-      sys.path.insert( GetStandardLibraryIndexInSysPath() + 1,
-                       os.path.realpath( os.path.join( path_to_third_party,
-                                                       folder ) ) )
-      continue
-
-    # The regex module is already included in SetUpPythonPath.
-    if folder == 'cregex':
-      continue
-
-    sys.path.insert( 0, os.path.realpath( os.path.join( path_to_third_party,
-                                                        folder ) ) )
+def SetUpPythonPath():
+  # python-future needs special handling. Not only does it store the modules
+  # under its 'src' folder, but SOME of its modules are only meant to be
+  # accessible under py2, not py3. This is because these modules (like
+  # `queue`) are implementations of modules present in the py3 standard
+  # library. Furthermore, we need to be sure that they are not overridden by
+  # already installed packages (for example, the 'builtins' module from
+  # 'pies2overrides' or a different version of 'python-future'). To work
+  # around these issues, we place the python-future just after the Python
+  # standard library so that its modules can be overridden by standard
+  # modules but not by installed packages.
+  sys.path.insert( GetStandardLibraryIndexInSysPath() + 1,
+                   p.join( DIR_OF_THIRD_PARTY, 'python-future', 'src' ) )
+  sys.path[ 0:0 ] = [ p.join( ROOT_DIR ),
+                      p.join( DIR_OF_THIRD_PARTY, 'bottle' ),
+                      p.join( DIR_OF_THIRD_PARTY, 'cregex',
+                              'regex_{}'.format( sys.version_info[ 0 ] ) ),
+                      p.join( DIR_OF_THIRD_PARTY, 'frozendict' ),
+                      p.join( DIR_OF_THIRD_PARTY, 'jedi' ),
+                      p.join( DIR_OF_THIRD_PARTY, 'parso' ),
+                      p.join( DIR_OF_THIRD_PARTY, 'requests' ),
+                      p.join( DIR_OF_THIRD_PARTY, 'waitress' ) ]

ycmd/tests/server_utils_test.py

@@ -22,35 +22,12 @@
 # Not installing aliases from python-future; it's unreliable and slow.
 from builtins import *  # noqa
 
-from hamcrest import ( assert_that, calling, contains, contains_inanyorder,
-                       empty, equal_to, has_length, raises )
+from hamcrest import assert_that, calling, empty, equal_to, has_length, raises
 from mock import patch
-from nose.tools import ok_
-import os.path
-import sys
-
-from ycmd.server_utils import ( AddNearestThirdPartyFoldersToSysPath,
-                                CompatibleWithCurrentCore,
-                                GetStandardLibraryIndexInSysPath,
-                                PathToNearestThirdPartyFolder )
-from ycmd.tests import PathToTestFile
 
-DIR_OF_THIRD_PARTY = os.path.abspath(
-  os.path.join( os.path.dirname( __file__ ), '..', '..', 'third_party' ) )
-THIRD_PARTY_FOLDERS = [
-  os.path.join( DIR_OF_THIRD_PARTY, 'bottle' ),
-  os.path.join( DIR_OF_THIRD_PARTY, 'frozendict' ),
-  os.path.join( DIR_OF_THIRD_PARTY, 'go' ),
-  os.path.join( DIR_OF_THIRD_PARTY, 'jedi' ),
-  os.path.join( DIR_OF_THIRD_PARTY, 'OmniSharpServer' ),
-  os.path.join( DIR_OF_THIRD_PARTY, 'parso' ),
-  os.path.join( DIR_OF_THIRD_PARTY, 'racerd' ),
-  os.path.join( DIR_OF_THIRD_PARTY, 'requests' ),
-  os.path.join( DIR_OF_THIRD_PARTY, 'tern_runtime' ),
-  os.path.join( DIR_OF_THIRD_PARTY, 'tsserver' ),
-  os.path.join( DIR_OF_THIRD_PARTY, 'waitress' ),
-  os.path.join( DIR_OF_THIRD_PARTY, 'eclipse.jdt.ls' ),
-]
+from ycmd.server_utils import ( CompatibleWithCurrentCore,
+                                GetStandardLibraryIndexInSysPath )
+from ycmd.tests import PathToTestFile
 
 
 @patch( 'ycmd.server_utils._logger', autospec = True )
@@ -155,70 +132,6 @@ def CompatibleWithCurrentCore_Outdated_NoVersionMatch_test( logger, *args ):
     'script. See the documentation for more details.' )
 
 
-def PathToNearestThirdPartyFolder_Success_test():
-  ok_( PathToNearestThirdPartyFolder( os.path.abspath( __file__ ) ) )
-
-
-def PathToNearestThirdPartyFolder_Failure_test():
-  ok_( not PathToNearestThirdPartyFolder( os.path.expanduser( '~' ) ) )
-
-
-def AddNearestThirdPartyFoldersToSysPath_Failure_test():
-  assert_that(
-    calling( AddNearestThirdPartyFoldersToSysPath ).with_args(
-      os.path.expanduser( '~' ) ),
-    raises( RuntimeError, '.*third_party folder.*' ) )
-
-
-@patch( 'sys.path', [
-  PathToTestFile( 'python-future', 'some', 'path' ),
-  PathToTestFile( 'python-future', 'standard_library' ),
-  PathToTestFile( 'python-future', 'standard_library', 'site-packages' ),
-  PathToTestFile( 'python-future', 'another', 'path' ) ] )
-def AddNearestThirdPartyFoldersToSysPath_FutureAfterStandardLibrary_test(
-  *args ):
-  AddNearestThirdPartyFoldersToSysPath( __file__ )
-  assert_that( sys.path[ : len( THIRD_PARTY_FOLDERS ) ], contains_inanyorder(
-    *THIRD_PARTY_FOLDERS
-  ) )
-  assert_that( sys.path[ len( THIRD_PARTY_FOLDERS ) : ], contains(
-    PathToTestFile( 'python-future', 'some', 'path' ),
-    PathToTestFile( 'python-future', 'standard_library' ),
-    os.path.join( DIR_OF_THIRD_PARTY, 'python-future', 'src' ),
-    PathToTestFile( 'python-future', 'standard_library', 'site-packages' ),
-    PathToTestFile( 'python-future', 'another', 'path' )
-  ) )
-
-
-@patch( 'sys.path', [
-  PathToTestFile( 'python-future', 'some', 'path' ),
-  PathToTestFile( 'python-future', 'another', 'path' ) ] )
-def AddNearestThirdPartyFoldersToSysPath_ErrorIfNoStandardLibrary_test( *args ):
-  assert_that(
-    calling( AddNearestThirdPartyFoldersToSysPath ).with_args( __file__ ),
-    raises( RuntimeError,
-            'Could not find standard library path in Python path.' ) )
-
-
-@patch( 'sys.path', [
-  PathToTestFile( 'python-future', 'some', 'path' ),
-  PathToTestFile( 'python-future', 'virtualenv_library' ),
-  PathToTestFile( 'python-future', 'standard_library' ),
-  PathToTestFile( 'python-future', 'another', 'path' ) ] )
-def AddNearestThirdPartyFoldersToSysPath_IgnoreVirtualEnvLibrary_test( *args ):
-  AddNearestThirdPartyFoldersToSysPath( __file__ )
-  assert_that( sys.path[ : len( THIRD_PARTY_FOLDERS ) ], contains_inanyorder(
-    *THIRD_PARTY_FOLDERS
-  ) )
-  assert_that( sys.path[ len( THIRD_PARTY_FOLDERS ) : ], contains(
-    PathToTestFile( 'python-future', 'some', 'path' ),
-    PathToTestFile( 'python-future', 'virtualenv_library' ),
-    PathToTestFile( 'python-future', 'standard_library' ),
-    os.path.join( DIR_OF_THIRD_PARTY, 'python-future', 'src' ),
-    PathToTestFile( 'python-future', 'another', 'path' )
-  ) )
-
-
 @patch( 'sys.path', [
   PathToTestFile( 'python-future', 'some', 'path' ),
   PathToTestFile( 'python-future', 'another', 'path' ) ] )