cli/operation/GetColumnsOperation.java

0001 /**
0002  * Licensed to the Apache Software Foundation (ASF) under one
0003  * or more contributor license agreements.  See the NOTICE file
0004  * distributed with this work for additional information
0005  * regarding copyright ownership.  The ASF licenses this file
0006  * to you under the Apache License, Version 2.0 (the
0007  * "License"); you may not use this file except in compliance
0008  * with the License.  You may obtain a copy of the License at
0009  *
0010  *     http://www.apache.org/licenses/LICENSE-2.0
0011  *
0012  * Unless required by applicable law or agreed to in writing, software
0013  * distributed under the License is distributed on an "AS IS" BASIS,
0014  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0015  * See the License for the specific language governing permissions and
0016  * limitations under the License.
0017  */
0018
0019 package org.apache.hive.service.cli.operation;
0020
0021 import java.sql.DatabaseMetaData;
0022 import java.util.ArrayList;
0023 import java.util.Collections;
0024 import java.util.HashMap;
0025 import java.util.HashSet;
0026 import java.util.List;
0027 import java.util.Map;
0028 import java.util.Map.Entry;
0029 import java.util.Set;
0030 import java.util.regex.Pattern;
0031
0032 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
0033 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
0034 import org.apache.hadoop.hive.metastore.api.PrimaryKeysRequest;
0035 import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
0036 import org.apache.hadoop.hive.metastore.api.Table;
0037 import org.apache.hadoop.hive.ql.metadata.TableIterable;
0038 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
0039 import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
0040 import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType;
0041 import org.apache.hadoop.hive.ql.session.SessionState;
0042 import org.apache.hadoop.hive.serde2.thrift.Type;
0043 import org.apache.hive.service.cli.ColumnDescriptor;
0044 import org.apache.hive.service.cli.FetchOrientation;
0045 import org.apache.hive.service.cli.HiveSQLException;
0046 import org.apache.hive.service.cli.OperationState;
0047 import org.apache.hive.service.cli.OperationType;
0048 import org.apache.hive.service.cli.RowSet;
0049 import org.apache.hive.service.cli.RowSetFactory;
0050 import org.apache.hive.service.cli.TableSchema;
0051 import org.apache.hive.service.cli.session.HiveSession;
0052
0053 /**
0054  * GetColumnsOperation.
0055  *
0056  */
0057 public class GetColumnsOperation extends MetadataOperation {
0058
0059   protected static final TableSchema RESULT_SET_SCHEMA = new TableSchema()
0060   .addPrimitiveColumn("TABLE_CAT", Type.STRING_TYPE,
0061       "Catalog name. NULL if not applicable")
0062   .addPrimitiveColumn("TABLE_SCHEM", Type.STRING_TYPE,
0063       "Schema name")
0064   .addPrimitiveColumn("TABLE_NAME", Type.STRING_TYPE,
0065       "Table name")
0066   .addPrimitiveColumn("COLUMN_NAME", Type.STRING_TYPE,
0067       "Column name")
0068   .addPrimitiveColumn("DATA_TYPE", Type.INT_TYPE,
0069       "SQL type from java.sql.Types")
0070   .addPrimitiveColumn("TYPE_NAME", Type.STRING_TYPE,
0071       "Data source dependent type name, for a UDT the type name is fully qualified")
0072   .addPrimitiveColumn("COLUMN_SIZE", Type.INT_TYPE,
0073       "Column size. For char or date types this is the maximum number of characters,"
0074       + " for numeric or decimal types this is precision.")
0075   .addPrimitiveColumn("BUFFER_LENGTH", Type.TINYINT_TYPE,
0076       "Unused")
0077   .addPrimitiveColumn("DECIMAL_DIGITS", Type.INT_TYPE,
0078       "The number of fractional digits")
0079   .addPrimitiveColumn("NUM_PREC_RADIX", Type.INT_TYPE,
0080       "Radix (typically either 10 or 2)")
0081   .addPrimitiveColumn("NULLABLE", Type.INT_TYPE,
0082       "Is NULL allowed")
0083   .addPrimitiveColumn("REMARKS", Type.STRING_TYPE,
0084       "Comment describing column (may be null)")
0085   .addPrimitiveColumn("COLUMN_DEF", Type.STRING_TYPE,
0086       "Default value (may be null)")
0087   .addPrimitiveColumn("SQL_DATA_TYPE", Type.INT_TYPE,
0088       "Unused")
0089   .addPrimitiveColumn("SQL_DATETIME_SUB", Type.INT_TYPE,
0090       "Unused")
0091   .addPrimitiveColumn("CHAR_OCTET_LENGTH", Type.INT_TYPE,
0092       "For char types the maximum number of bytes in the column")
0093   .addPrimitiveColumn("ORDINAL_POSITION", Type.INT_TYPE,
0094       "Index of column in table (starting at 1)")
0095   .addPrimitiveColumn("IS_NULLABLE", Type.STRING_TYPE,
0096       "\"NO\" means column definitely does not allow NULL values; "
0097       + "\"YES\" means the column might allow NULL values. An empty "
0098       + "string means nobody knows.")
0099   .addPrimitiveColumn("SCOPE_CATALOG", Type.STRING_TYPE,
0100       "Catalog of table that is the scope of a reference attribute "
0101       + "(null if DATA_TYPE isn't REF)")
0102   .addPrimitiveColumn("SCOPE_SCHEMA", Type.STRING_TYPE,
0103       "Schema of table that is the scope of a reference attribute "
0104       + "(null if the DATA_TYPE isn't REF)")
0105   .addPrimitiveColumn("SCOPE_TABLE", Type.STRING_TYPE,
0106       "Table name that this the scope of a reference attribure "
0107       + "(null if the DATA_TYPE isn't REF)")
0108   .addPrimitiveColumn("SOURCE_DATA_TYPE", Type.SMALLINT_TYPE,
0109       "Source type of a distinct type or user-generated Ref type, "
0110       + "SQL type from java.sql.Types (null if DATA_TYPE isn't DISTINCT or user-generated REF)")
0111   .addPrimitiveColumn("IS_AUTO_INCREMENT", Type.STRING_TYPE,
0112       "Indicates whether this column is auto incremented.");
0113
0114   private final String catalogName;
0115   private final String schemaName;
0116   private final String tableName;
0117   private final String columnName;
0118
0119   protected final RowSet rowSet;
0120
0121   protected GetColumnsOperation(HiveSession parentSession, String catalogName, String schemaName,
0122       String tableName, String columnName) {
0123     super(parentSession, OperationType.GET_COLUMNS);
0124     this.catalogName = catalogName;
0125     this.schemaName = schemaName;
0126     this.tableName = tableName;
0127     this.columnName = columnName;
0128     this.rowSet = RowSetFactory.create(RESULT_SET_SCHEMA, getProtocolVersion(), false);
0129   }
0130
0131   @Override
0132   public void runInternal() throws HiveSQLException {
0133     setState(OperationState.RUNNING);
0134     try {
0135       IMetaStoreClient metastoreClient = getParentSession().getMetaStoreClient();
0136       String schemaPattern = convertSchemaPattern(schemaName);
0137       String tablePattern = convertIdentifierPattern(tableName, true);
0138
0139       Pattern columnPattern = null;
0140       if (columnName != null) {
0141         columnPattern = Pattern.compile(convertIdentifierPattern(columnName, false));
0142       }
0143
0144       List<String> dbNames = metastoreClient.getDatabases(schemaPattern);
0145       Collections.sort(dbNames);
0146       Map<String, List<String>> db2Tabs = new HashMap<>();
0147
0148       for (String dbName : dbNames) {
0149         List<String> tableNames = metastoreClient.getTables(dbName, tablePattern);
0150         Collections.sort(tableNames);
0151         db2Tabs.put(dbName, tableNames);
0152       }
0153
0154       if (isAuthV2Enabled()) {
0155         List<HivePrivilegeObject> privObjs = getPrivObjs(db2Tabs);
0156         String cmdStr = "catalog : " + catalogName + ", schemaPattern : " + schemaName
0157             + ", tablePattern : " + tableName;
0158         authorizeMetaGets(HiveOperationType.GET_COLUMNS, privObjs, cmdStr);
0159       }
0160
0161       int maxBatchSize = SessionState.get().getConf().getIntVar(ConfVars.METASTORE_BATCH_RETRIEVE_MAX);
0162       for (Entry<String, List<String>> dbTabs : db2Tabs.entrySet()) {
0163         String dbName = dbTabs.getKey();
0164         List<String> tableNames = dbTabs.getValue();
0165
0166         for (Table table : new TableIterable(metastoreClient, dbName, tableNames, maxBatchSize)) {
0167
0168           TableSchema schema = new TableSchema(metastoreClient.getSchema(dbName,
0169               table.getTableName()));
0170           List<SQLPrimaryKey> primaryKeys = metastoreClient.getPrimaryKeys(new PrimaryKeysRequest(dbName, table.getTableName()));
0171           Set<String> pkColNames = new HashSet<>();
0172           for(SQLPrimaryKey key : primaryKeys) {
0173             pkColNames.add(key.getColumn_name().toLowerCase());
0174           }
0175           for (ColumnDescriptor column : schema.getColumnDescriptors()) {
0176             if (columnPattern != null && !columnPattern.matcher(column.getName()).matches()) {
0177               continue;
0178             }
0179             Object[] rowData = new Object[] {
0180                 null,  // TABLE_CAT
0181                 table.getDbName(), // TABLE_SCHEM
0182                 table.getTableName(), // TABLE_NAME
0183                 column.getName(), // COLUMN_NAME
0184                 column.getType().toJavaSQLType(), // DATA_TYPE
0185                 column.getTypeName(), // TYPE_NAME
0186                 column.getTypeDescriptor().getColumnSize(), // COLUMN_SIZE
0187                 null, // BUFFER_LENGTH, unused
0188                 column.getTypeDescriptor().getDecimalDigits(), // DECIMAL_DIGITS
0189                 column.getType().getNumPrecRadix(), // NUM_PREC_RADIX
0190                 pkColNames.contains(column.getName().toLowerCase()) ? DatabaseMetaData.columnNoNulls
0191                     : DatabaseMetaData.columnNullable, // NULLABLE
0192                 column.getComment(), // REMARKS
0193                 null, // COLUMN_DEF
0194                 null, // SQL_DATA_TYPE
0195                 null, // SQL_DATETIME_SUB
0196                 null, // CHAR_OCTET_LENGTH
0197                 column.getOrdinalPosition(), // ORDINAL_POSITION
0198                 pkColNames.contains(column.getName().toLowerCase()) ? "NO" : "YES", // IS_NULLABLE
0199                 null, // SCOPE_CATALOG
0200                 null, // SCOPE_SCHEMA
0201                 null, // SCOPE_TABLE
0202                 null, // SOURCE_DATA_TYPE
0203                 "NO", // IS_AUTO_INCREMENT
0204             };
0205             rowSet.addRow(rowData);
0206           }
0207         }
0208       }
0209       setState(OperationState.FINISHED);
0210     } catch (Exception e) {
0211       setState(OperationState.ERROR);
0212       throw new HiveSQLException(e);
0213     }
0214
0215   }
0216
0217
0218   private List<HivePrivilegeObject> getPrivObjs(Map<String, List<String>> db2Tabs) {
0219     List<HivePrivilegeObject> privObjs = new ArrayList<>();
0220     for (Entry<String, List<String>> dbTabs : db2Tabs.entrySet()) {
0221       for (String tabName : dbTabs.getValue()) {
0222         privObjs.add(new HivePrivilegeObject(HivePrivilegeObjectType.TABLE_OR_VIEW, dbTabs.getKey(),
0223             tabName));
0224       }
0225     }
0226     return privObjs;
0227   }
0228
0229   /* (non-Javadoc)
0230    * @see org.apache.hive.service.cli.Operation#getResultSetSchema()
0231    */
0232   @Override
0233   public TableSchema getResultSetSchema() throws HiveSQLException {
0234     assertState(OperationState.FINISHED);
0235     return RESULT_SET_SCHEMA;
0236   }
0237
0238   /* (non-Javadoc)
0239    * @see org.apache.hive.service.cli.Operation#getNextRowSet(org.apache.hive.service.cli.FetchOrientation, long)
0240    */
0241   @Override
0242   public RowSet getNextRowSet(FetchOrientation orientation, long maxRows) throws HiveSQLException {
0243     assertState(OperationState.FINISHED);
0244     validateDefaultFetchOrientation(orientation);
0245     if (orientation.equals(FetchOrientation.FETCH_FIRST)) {
0246       rowSet.setStartOffset(0);
0247     }
0248     return rowSet.extractSubset((int)maxRows);
0249   }
0250
0251 }