'Flink Avro Error Expecting type to be a PojoTypeInfo
I'm trying to convert a HashMap<String, Object> to an Avro record. I get this runtime exception when I do a
DataStream<AvroRecord> dsRpvSchema = filteredVlfRPV.flatMap(new MessageFlattener()) .name("ToAvroSchema").uid("ToAvroSchema").startNewChain();
Not sure what the issue is with Flink converting the valid avro record to Flink Avro.
java.lang.IllegalStateException: Expecting type to be a PojoTypeInfo
at org.apache.flink.formats.avro.typeutils.AvroTypeInfo.generateFieldsFromAvroSchema(AvroTypeInfo.java:71)
at org.apache.flink.formats.avro.typeutils.AvroTypeInfo.<init>(AvroTypeInfo.java:55)
at org.apache.flink.formats.avro.utils.AvroKryoSerializerUtils.createAvroTypeInfo(AvroKryoSerializerUtils.java:81)
at org.apache.flink.api.java.typeutils.TypeExtractor.privateGetForClass(TypeExtractor.java:1653)
at org.apache.flink.api.java.typeutils.TypeExtractor.privateGetForClass(TypeExtractor.java:1559)
at org.apache.flink.api.java.typeutils.TypeExtractor.createTypeInfoWithTypeHierarchy(TypeExtractor.java:866)
at org.apache.flink.api.java.typeutils.TypeExtractor.privateCreateTypeInfo(TypeExtractor.java:747)
at org.apache.flink.api.java.typeutils.TypeExtractor.getUnaryOperatorReturnType(TypeExtractor.java:531)
at org.apache.flink.api.java.typeutils.TypeExtractor.getFlatMapReturnTypes(TypeExtractor.java:168)
at org.apache.flink.streaming.api.datastream.DataStream.flatMap(DataStream.java:637)
I do see two questions on this but no answers provided:
- Flink Kafka : Expecting type to be a PojoTypeInfo
- Deserialize Avro from kafka as SpecificRecord Failing. Expecting type to be a PojoTypeInfo
UPDATE: 02/01/2022. Below is the AvroRecord class definition.
import org.apache.avro.generic.GenericArray;
import org.apache.avro.specific.SpecificData;
import org.apache.avro.util.Utf8;
import org.apache.avro.message.BinaryMessageEncoder;
import org.apache.avro.message.BinaryMessageDecoder;
import org.apache.avro.message.SchemaStore;
@org.apache.avro.specific.AvroGenerated
public class AvroRecord extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
private static final long serialVersionUID = 9071485731787422200L;
public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"AvroRecord\",\"namespace\":\"com.sample.dataplatform.stream.avro\",\"fields\":[{\"name\":\"_id\",\"type\":\"string\",\"doc\":\"car_id\"}]}");
public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }
private static SpecificData MODEL$ = new SpecificData();
private static final BinaryMessageEncoder<AvroRecord> ENCODER =
new BinaryMessageEncoder<AvroRecord>(MODEL$, SCHEMA$);
private static final BinaryMessageDecoder<AvroRecord> DECODER =
new BinaryMessageDecoder<AvroRecord>(MODEL$, SCHEMA$);
/**
* Return the BinaryMessageEncoder instance used by this class.
* @return the message encoder used by this class
*/
public static BinaryMessageEncoder<AvroRecord> getEncoder() {
return ENCODER;
}
/**
* Return the BinaryMessageDecoder instance used by this class.
* @return the message decoder used by this class
*/
public static BinaryMessageDecoder<AvroRecord> getDecoder() {
return DECODER;
}
/**
* Create a new BinaryMessageDecoder instance for this class that uses the specified {@link SchemaStore}.
* @param resolver a {@link SchemaStore} used to find schemas by fingerprint
* @return a BinaryMessageDecoder instance for this class backed by the given SchemaStore
*/
public static BinaryMessageDecoder<AvroRecord> createDecoder(SchemaStore resolver) {
return new BinaryMessageDecoder<AvroRecord>(MODEL$, SCHEMA$, resolver);
}
/**
* Serializes this AvroRecord to a ByteBuffer.
* @return a buffer holding the serialized data for this instance
* @throws java.io.IOException if this instance could not be serialized
*/
public java.nio.ByteBuffer toByteBuffer() throws java.io.IOException {
return ENCODER.encode(this);
}
/**
* Deserializes a AvroRecord from a ByteBuffer.
* @param b a byte buffer holding serialized data for an instance of this class
* @return a AvroRecord instance decoded from the given buffer
* @throws java.io.IOException if the given bytes could not be deserialized into an instance of this class
*/
public static AvroRecord fromByteBuffer(
java.nio.ByteBuffer b) throws java.io.IOException {
return DECODER.decode(b);
}
/** car_id */
private java.lang.CharSequence _id;
/**
* Default constructor. Note that this does not initialize fields
* to their default values from the schema. If that is desired then
* one should use <code>newBuilder()</code>.
*/
public AvroRecord() {}
/**
* All-args constructor.
* @param _id car_id
*/
public AvroRecord(java.lang.CharSequence _id) {
this._id = _id;
}
public org.apache.avro.specific.SpecificData getSpecificData() { return MODEL$; }
public org.apache.avro.Schema getSchema() { return SCHEMA$; }
// Used by DatumWriter. Applications should not call.
public java.lang.Object get(int field$) {
switch (field$) {
case 0: return _id;
default: throw new IndexOutOfBoundsException("Invalid index: " + field$);
}
}
// Used by DatumReader. Applications should not call.
@SuppressWarnings(value="unchecked")
public void put(int field$, java.lang.Object value$) {
switch (field$) {
case 0: _id = (java.lang.CharSequence)value$; break;
default: throw new IndexOutOfBoundsException("Invalid index: " + field$);
}
}
/**
* Gets the value of the '_id' field.
* @return car_id
*/
public java.lang.CharSequence getId$1() {
return _id;
}
/**
* Sets the value of the '_id' field.
* car_id
* @param value the value to set.
*/
public void setId$1(java.lang.CharSequence value) {
this._id = value;
}
/**
* Creates a new AvroRecord RecordBuilder.
* @return A new AvroRecord RecordBuilder
*/
public static com.sample.dataplatform.stream.avro.AvroRecord.Builder newBuilder() {
return new com.sample.dataplatform.stream.avro.AvroRecord.Builder();
}
/**
* Creates a new AvroRecord RecordBuilder by copying an existing Builder.
* @param other The existing builder to copy.
* @return A new AvroRecord RecordBuilder
*/
public static com.sample.dataplatform.stream.avro.AvroRecord.Builder newBuilder(com.sample.dataplatform.stream.avro.AvroRecord.Builder other) {
if (other == null) {
return new com.sample.dataplatform.stream.avro.AvroRecord.Builder();
} else {
return new com.sample.dataplatform.stream.avro.AvroRecord.Builder(other);
}
}
/**
* Creates a new AvroRecord RecordBuilder by copying an existing AvroRecord instance.
* @param other The existing instance to copy.
* @return A new AvroRecord RecordBuilder
*/
public static com.sample.dataplatform.stream.avro.AvroRecord.Builder newBuilder(com.sample.dataplatform.stream.avro.AvroRecord other) {
if (other == null) {
return new com.sample.dataplatform.stream.avro.AvroRecord.Builder();
} else {
return new com.sample.dataplatform.stream.avro.AvroRecord.Builder(other);
}
}
/**
* RecordBuilder for AvroRecord instances.
*/
@org.apache.avro.specific.AvroGenerated
public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<AvroRecord>
implements org.apache.avro.data.RecordBuilder<AvroRecord> {
/** car_id */
private java.lang.CharSequence _id;
/** Creates a new Builder */
private Builder() {
super(SCHEMA$);
}
/**
* Creates a Builder by copying an existing Builder.
* @param other The existing Builder to copy.
*/
private Builder(com.sample.dataplatform.stream.avro.AvroRecord.Builder other) {
super(other);
if (isValidValue(fields()[0], other._id)) {
this._id = data().deepCopy(fields()[0].schema(), other._id);
fieldSetFlags()[0] = other.fieldSetFlags()[0];
}
}
/**
* Creates a Builder by copying an existing AvroRecord instance
* @param other The existing instance to copy.
*/
private Builder(com.sample.dataplatform.stream.avro.AvroRecord other) {
super(SCHEMA$);
if (isValidValue(fields()[0], other._id)) {
this._id = data().deepCopy(fields()[0].schema(), other._id);
fieldSetFlags()[0] = true;
}
}
/**
* Gets the value of the '_id' field.
* car_id
* @return The value.
*/
public java.lang.CharSequence getId$1() {
return _id;
}
/**
* Sets the value of the '_id' field.
* car_id
* @param value The value of '_id'.
* @return This builder.
*/
public com.sample.dataplatform.stream.avro.AvroRecord.Builder setId$1(java.lang.CharSequence value) {
validate(fields()[0], value);
this._id = value;
fieldSetFlags()[0] = true;
return this;
}
/**
* Checks whether the '_id' field has been set.
* car_id
* @return True if the '_id' field has been set, false otherwise.
*/
public boolean hasId$1() {
return fieldSetFlags()[0];
}
/**
* Clears the value of the '_id' field.
* car_id
* @return This builder.
*/
public com.sample.dataplatform.stream.avro.AvroRecord.Builder clearId$1() {
_id = null;
fieldSetFlags()[0] = false;
return this;
}
@Override
@SuppressWarnings("unchecked")
public AvroRecord build() {
try {
AvroRecord record = new AvroRecord();
record._id = fieldSetFlags()[0] ? this._id : (java.lang.CharSequence) defaultValue(fields()[0]);
return record;
} catch (org.apache.avro.AvroMissingFieldException e) {
throw e;
} catch (java.lang.Exception e) {
throw new org.apache.avro.AvroRuntimeException(e);
}
}
}
@SuppressWarnings("unchecked")
private static final org.apache.avro.io.DatumWriter<AvroRecord>
WRITER$ = (org.apache.avro.io.DatumWriter<AvroRecord>)MODEL$.createDatumWriter(SCHEMA$);
@Override public void writeExternal(java.io.ObjectOutput out)
throws java.io.IOException {
WRITER$.write(this, SpecificData.getEncoder(out));
}
@SuppressWarnings("unchecked")
private static final org.apache.avro.io.DatumReader<AvroRecord>
READER$ = (org.apache.avro.io.DatumReader<AvroRecord>)MODEL$.createDatumReader(SCHEMA$);
@Override public void readExternal(java.io.ObjectInput in)
throws java.io.IOException {
READER$.read(this, SpecificData.getDecoder(in));
}
@Override protected boolean hasCustomCoders() { return true; }
@Override public void customEncode(org.apache.avro.io.Encoder out)
throws java.io.IOException
{
out.writeString(this._id);
}
@Override public void customDecode(org.apache.avro.io.ResolvingDecoder in)
throws java.io.IOException
{
org.apache.avro.Schema.Field[] fieldOrder = in.readFieldOrderIfDiff();
if (fieldOrder == null) {
this._id = in.readString(this._id instanceof Utf8 ? (Utf8)this._id : null);
} else {
for (int i = 0; i < 1; i++) {
switch (fieldOrder[i].pos()) {
case 0:
this._id = in.readString(this._id instanceof Utf8 ? (Utf8)this._id : null);
break;
default:
throw new java.io.IOException("Corrupt ResolvingDecoder.");
}
}
}
}
}
The AVRO schema (.avsc) file looks like this:
{
"namespace": "com.sample.dataplatform.stream.avro",
"type": "record",
"name": "AvroRecord",
"fields": [
{
"name": "_id",
"type": "string",
"doc": "car_id"
}
]
}
Any help is appreciated!
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|