|
||||
0001 # 0002 # Licensed to the Apache Software Foundation (ASF) under one or more 0003 # contributor license agreements. See the NOTICE file distributed with 0004 # this work for additional information regarding copyright ownership. 0005 # The ASF licenses this file to You under the Apache License, Version 2.0 0006 # (the "License"); you may not use this file except in compliance with 0007 # the License. You may obtain a copy of the License at 0008 # 0009 # http://www.apache.org/licenses/LICENSE-2.0 0010 # 0011 # Unless required by applicable law or agreed to in writing, software 0012 # distributed under the License is distributed on an "AS IS" BASIS, 0013 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 0014 # See the License for the specific language governing permissions and 0015 # limitations under the License. 0016 # 0017 0018 from __future__ import print_function 0019 0020 import sys 0021 0022 from pyspark.sql import SparkSession 0023 from pyspark.sql.types import StringType 0024 0025 0026 if __name__ == "__main__": 0027 """ 0028 Usage: pyfiles [major_python_version] 0029 """ 0030 spark = SparkSession \ 0031 .builder \ 0032 .appName("PyFilesTest") \ 0033 .getOrCreate() 0034 0035 from py_container_checks import version_check 0036 # Begin of Python container checks 0037 version_check(sys.argv[1], 2 if sys.argv[1] == "python" else 3) 0038 0039 # Check python executable at executors 0040 spark.udf.register("get_sys_ver", 0041 lambda: "%d.%d" % sys.version_info[:2], StringType()) 0042 [row] = spark.sql("SELECT get_sys_ver()").collect() 0043 driver_version = "%d.%d" % sys.version_info[:2] 0044 print("Python runtime version check for executor is: " + str(row[0] == driver_version)) 0045 0046 spark.stop()
[ Source navigation ] | [ Diff markup ] | [ Identifier search ] | [ general search ] |
This page was automatically generated by the 2.1.0 LXR engine. The LXR team |