Java org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory 代码实例

・18 分钟阅读

以下是展示如何使用org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory的最佳示例。 我们使用了代码质量辨别算法从开源项目中提取出了最佳的优秀示例。

实例 1


@Test
public void testInitialize() throws UDFArgumentException {
    PerceptronUDTF udtf = new PerceptronUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI = ObjectInspectorFactory.getStandardListObjectInspector(intOI);
    /* test for INT_TYPE_NAME feature */
    StructObjectInspector intListSOI = udtf.initialize(new ObjectInspector[] {
            intListOI, intOI });
    assertEquals("struct<feature:int,weight:float>", intListSOI.getTypeName());
    /* test for STRING_TYPE_NAME feature */
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI = ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    StructObjectInspector stringListSOI = udtf.initialize(new ObjectInspector[] {
            stringListOI, intOI });
    assertEquals("struct<feature:string,weight:float>", stringListSOI.getTypeName());
    /* test for BIGINT_TYPE_NAME feature */
    ObjectInspector longOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
    ListObjectInspector longListOI = ObjectInspectorFactory.getStandardListObjectInspector(longOI);
    StructObjectInspector longListSOI = udtf.initialize(new ObjectInspector[] {
            longListOI, intOI });
    assertEquals("struct<feature:bigint,weight:float>", longListSOI.getTypeName());
}
 

实例 2


@Override
public ObjectInspector initialize(ObjectInspector[] arg0)
        throws UDFArgumentException {
    if (arg0.length < 2) {
        throw new UDFArgumentException("combine_sketch takes at least two arguments; a set of array<string> or a set of array<bigint>");
    }
    if (arg0[0].getCategory() != Category.LIST) {
        throw new UDFArgumentException("combine_sketch takes at least two arguments; a set of array<string> or a set of array<bigint>");
    }
    ObjectInspector lastInspector = arg0[arg0.length - 1];
    int listLen = arg0.length;
    if (lastInspector.getCategory() == Category.PRIMITIVE
            && ((PrimitiveObjectInspector) lastInspector).getPrimitiveCategory() == PrimitiveCategory.INT) {
        if (lastInspector instanceof ConstantObjectInspector) {
        } else {
            throw new UDFArgumentException(" Sketch set size must an integer");
        }
    }
    this.listInspectors = new ListObjectInspector[arg0.length];
    this.listInspectors[0] = (ListObjectInspector) arg0[0];
    if (this.listInspectors[0].getListElementObjectInspector().getCategory() != Category.PRIMITIVE) {
        throw new UDFArgumentException("combine_sketch takes at least two arguments; a set of array<string> or a set of array<bigint>");
    }
    this.elemCategory = ((PrimitiveObjectInspector) ((listInspectors[0].getListElementObjectInspector()))).getPrimitiveCategory();
    if (this.elemCategory != PrimitiveCategory.STRING && this.elemCategory != PrimitiveCategory.LONG) {
        throw new UDFArgumentException("combine_sketch takes at least two arguments; a set of array<string> or a set of array<bigint>");
    }
    for (int i = 1; i < arg0.length; ++i) {
        if (arg0[i].getCategory() != Category.LIST) {
            throw new UDFArgumentException("combine_sketch takes at least two arguments; a set of array<string> or a set of array<bigint>");
        }
        this.listInspectors[i] = (ListObjectInspector) arg0[i];
        if (((PrimitiveObjectInspector) ((listInspectors[0].getListElementObjectInspector()))).getPrimitiveCategory() != elemCategory) {
            throw new UDFArgumentException("combine_sketch takes at least two arguments; a set of array<string> or a set of array<bigint>");
        }
    }
    return ObjectInspectorFactory.getStandardListObjectInspector(
            PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(elemCategory));
}
 

实例 3


@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
    if(argOIs.length != 2 && argOIs.length != 3) {
        throw new UDFArgumentException("vm_tree_predict takes 2 or 3 arguments");
    }
    if(HiveUtils.isStringOI(argOIs[0]) == false) {
        throw new UDFArgumentException("first argument is expected to be string but unexpected type was detected: "
                + TypeInfoUtils.getTypeInfoFromObjectInspector(argOIs[0]));
    }
    ListObjectInspector listOI = HiveUtils.asListOI(argOIs[1]);
    this.featureListOI = listOI;
    ObjectInspector elemOI = listOI.getListElementObjectInspector();
    this.featureElemOI = HiveUtils.asDoubleCompatibleOI(elemOI);
    boolean classification = false;
    if(argOIs.length == 3) {
        classification = HiveUtils.getConstBoolean(argOIs[2]);
    }
    this.classification = classification;
    if(classification) {
        return PrimitiveObjectInspectorFactory.writableIntObjectInspector;
    } else {
        return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
    }
}
 

实例 4


@Override
public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {
    assert (parameters.length == 1);
    super.init(mode, parameters);
    // initialize input
    if(mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data
        this.inputListOI = (ListObjectInspector) parameters[0];
        this.inputListElemOI = HiveUtils.asDoubleCompatibleOI(inputListOI.getListElementObjectInspector());
    } else {// from partial aggregation
        StructObjectInspector soi = (StructObjectInspector) parameters[0];
        this.internalMergeOI = soi;
        this.sizeField = soi.getStructFieldRef("size");
        this.sumField = soi.getStructFieldRef("sum");
        this.countField = soi.getStructFieldRef("count");
        this.sizeOI = PrimitiveObjectInspectorFactory.writableIntObjectInspector;
        this.sumOI = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
        this.countOI = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
    }
    // initialize output
    final ObjectInspector outputOI;
    if(mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial
        outputOI = internalMergeOI();
    } else {// terminate
        outputOI = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
    }
    return outputOI;
}
 

实例 5


@Test
public void testInitialize() throws UDFArgumentException {
    AdaGradUDTF udtf = new AdaGradUDTF();
    ObjectInspector labelOI = PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI = ObjectInspectorFactory.getStandardListObjectInspector(intOI);
    /* test for INT_TYPE_NAME feature */
    StructObjectInspector intListSOI = udtf.initialize(new ObjectInspector[] { intListOI,
            labelOI });
    assertEquals("struct<feature:int,weight:float>", intListSOI.getTypeName());
    /* test for STRING_TYPE_NAME feature */
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI = ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    StructObjectInspector stringListSOI = udtf.initialize(new ObjectInspector[] { stringListOI,
            labelOI });
    assertEquals("struct<feature:string,weight:float>", stringListSOI.getTypeName());
    /* test for BIGINT_TYPE_NAME feature */
    ObjectInspector longOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
    ListObjectInspector longListOI = ObjectInspectorFactory.getStandardListObjectInspector(longOI);
    StructObjectInspector longListSOI = udtf.initialize(new ObjectInspector[] { longListOI,
            labelOI });
    assertEquals("struct<feature:bigint,weight:float>", longListSOI.getTypeName());
}
 

实例 6


private ObjectInspector getObjectInspector(final TypeInfo typeInfo) {
  if (typeInfo.equals(TypeInfoFactory.doubleTypeInfo)) {
    return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
  } else if (typeInfo.equals(TypeInfoFactory.booleanTypeInfo)) {
    return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector;
  } else if (typeInfo.equals(TypeInfoFactory.floatTypeInfo)) {
    return PrimitiveObjectInspectorFactory.writableFloatObjectInspector;
  } else if (typeInfo.equals(TypeInfoFactory.intTypeInfo)) {
    return PrimitiveObjectInspectorFactory.writableIntObjectInspector;
  } else if (typeInfo.equals(TypeInfoFactory.longTypeInfo)) {
    return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
  } else if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) {
    return ParquetPrimitiveInspectorFactory.parquetStringInspector;
  } else if (typeInfo.getCategory().equals(Category.STRUCT)) {
    return new ArrayWritableObjectInspector((StructTypeInfo) typeInfo);
  } else if (typeInfo.getCategory().equals(Category.LIST)) {
    final TypeInfo subTypeInfo = ((ListTypeInfo) typeInfo).getListElementTypeInfo();
    return new ParquetHiveArrayInspector(getObjectInspector(subTypeInfo));
  } else if (typeInfo.getCategory().equals(Category.MAP)) {
    final TypeInfo keyTypeInfo = ((MapTypeInfo) typeInfo).getMapKeyTypeInfo();
    final TypeInfo valueTypeInfo = ((MapTypeInfo) typeInfo).getMapValueTypeInfo();
    if (keyTypeInfo.equals(TypeInfoFactory.stringTypeInfo) || keyTypeInfo.equals(TypeInfoFactory.byteTypeInfo)
            || keyTypeInfo.equals(TypeInfoFactory.shortTypeInfo)) {
      return new DeepParquetHiveMapInspector(getObjectInspector(keyTypeInfo), getObjectInspector(valueTypeInfo));
    } else {
      return new StandardParquetHiveMapInspector(getObjectInspector(keyTypeInfo), getObjectInspector(valueTypeInfo));
    }
  } else if (typeInfo.equals(TypeInfoFactory.timestampTypeInfo)) {
    throw new UnsupportedOperationException("timestamp not implemented yet");
  } else if (typeInfo.equals(TypeInfoFactory.byteTypeInfo)) {
    return ParquetPrimitiveInspectorFactory.parquetByteInspector;
  } else if (typeInfo.equals(TypeInfoFactory.shortTypeInfo)) {
    return ParquetPrimitiveInspectorFactory.parquetShortInspector;
  } else {
    throw new IllegalArgumentException("Unknown field info: " + typeInfo);
  }
}
 

实例 7


@BeforeClass
public static void setupClass() {
  binaryType = new PrimitiveTypeInfo();
  binaryType.setTypeName("binary");
  intType = new PrimitiveTypeInfo();
  intType.setTypeName("int");
  doubleType = new PrimitiveTypeInfo();
  doubleType.setTypeName("double");
  inputOI = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY);
  sketchSizeOI = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(intType,
      new IntWritable(1024));
  List<ObjectInspector> fields = new ArrayList<>();
  fields.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.INT));
  fields.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveCategory.BINARY));
  List<String> fieldNames = new ArrayList<>();
  fieldNames.add("sketchSize");
  fieldNames.add("sketch");
  intermediateStructType = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fields);
}
 

实例 8


@Override
public ObjectInspector initialize(@Nonnull final ObjectInspector[] argOIs)
        throws UDFArgumentException {
    final int numArgOIs = argOIs.length;
    if(numArgOIs < 2) {
        throw new UDFArgumentException("argOIs.length must be greater that or equals to 2: "
                + numArgOIs);
    }
    this.featureNames = HiveUtils.getConstStringArray(argOIs[0]);
    if(featureNames == null) {
        throw new UDFArgumentException("#featureNames should not be null");
    }
    int numFeatureNames = featureNames.length;
    if(numFeatureNames < 1) {
        throw new UDFArgumentException("#featureNames must be greater than or equals to 1: "
                + numFeatureNames);
    }
    int numFeatures = numArgOIs - 1;
    if(numFeatureNames != numFeatures) {
        throw new UDFArgumentException("#featureNames '" + numFeatureNames
                + "' != #arguments '" + numFeatures + "'");
    }
    this.inputOIs = new PrimitiveObjectInspector[numFeatures];
    for(int i = 0; i < numFeatures; i++) {
        ObjectInspector oi = argOIs[i + 1];
        inputOIs[i] = HiveUtils.asPrimitiveObjectInspector(oi);
    }
    this.result = new ArrayList<Text>(numFeatures);
    return ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
}
 

实例 9


@Test
public void testBooleanWeight() throws HiveException, IOException {
    VectorizeFeaturesUDF udf = new VectorizeFeaturesUDF();
    ObjectInspector[] argOIs = new ObjectInspector[3];
    List<String> featureNames = Arrays.asList("a", "b");
    argOIs[0] = ObjectInspectorFactory.getStandardConstantListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames);
    argOIs[1] = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    argOIs[2] = PrimitiveObjectInspectorFactory.javaBooleanObjectInspector;
    udf.initialize(argOIs);
    DeferredObject[] arguments = new DeferredObject[3];
    arguments[1] = new DeferredJavaObject(new Double(0.1));
    arguments[2] = new DeferredJavaObject(new Boolean(false));
    List<Text> actuals = udf.evaluate(arguments);
    //System.out.println(actuals);        
    List<Text> expected = WritableUtils.val(new String[] { "a:0.1" });
    Assert.assertEquals(expected, actuals);
    arguments[2] = new DeferredJavaObject(new Boolean(true));
    actuals = udf.evaluate(arguments);
    //System.out.println(actuals);
    expected = WritableUtils.val("a:0.1", "b:1.0");
    Assert.assertEquals(expected, actuals);
    udf.close();
}
 
讨论
淘淘あ西西 profile image