0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018 import os
0019
0020
0021 __all__ = ['SparkFiles']
0022
0023
0024 class SparkFiles(object):
0025
0026 """
0027 Resolves paths to files added through :meth:`SparkContext.addFile`.
0028
0029 SparkFiles contains only classmethods; users should not create SparkFiles
0030 instances.
0031 """
0032
0033 _root_directory = None
0034 _is_running_on_worker = False
0035 _sc = None
0036
0037 def __init__(self):
0038 raise NotImplementedError("Do not construct SparkFiles objects")
0039
0040 @classmethod
0041 def get(cls, filename):
0042 """
0043 Get the absolute path of a file added through :meth:`SparkContext.addFile`.
0044 """
0045 path = os.path.join(SparkFiles.getRootDirectory(), filename)
0046 return os.path.abspath(path)
0047
0048 @classmethod
0049 def getRootDirectory(cls):
0050 """
0051 Get the root directory that contains files added through
0052 :meth:`SparkContext.addFile`.
0053 """
0054 if cls._is_running_on_worker:
0055 return cls._root_directory
0056 else:
0057
0058 return cls._sc._jvm.org.apache.spark.SparkFiles.getRootDirectory()