0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018 from __future__ import print_function
0019
0020 import resource
0021 import sys
0022
0023 from pyspark.sql import SparkSession
0024
0025
0026 if __name__ == "__main__":
0027 """
0028 Usage: worker_memory_check [Memory_in_Mi]
0029 """
0030 spark = SparkSession \
0031 .builder \
0032 .appName("PyMemoryTest") \
0033 .getOrCreate()
0034 sc = spark.sparkContext
0035 if len(sys.argv) < 2:
0036 print("Usage: worker_memory_check [Memory_in_Mi]", file=sys.stderr)
0037 sys.exit(-1)
0038
0039 def f(x):
0040 rLimit = resource.getrlimit(resource.RLIMIT_AS)
0041 print("RLimit is " + str(rLimit))
0042 return rLimit
0043 resourceValue = sc.parallelize([1]).map(f).collect()[0][0]
0044 print("Resource Value is " + str(resourceValue))
0045 truthCheck = (resourceValue == int(sys.argv[1]))
0046 print("PySpark Worker Memory Check is: " + str(truthCheck))
0047 spark.stop()