public abstract class AbstractHdfsBolt extends BaseRichBolt
Modifier and Type | Field and Description |
---|---|
protected OutputCollector |
collector |
protected String |
configKey |
protected FileNameFormat |
fileNameFormat |
protected Integer |
fileRetryCount |
protected org.apache.hadoop.fs.FileSystem |
fs |
protected String |
fsUrl |
protected org.apache.hadoop.conf.Configuration |
hdfsConfig |
protected long |
offset |
protected int |
rotation |
protected ArrayList<RotationAction> |
rotationActions |
protected FileRotationPolicy |
rotationPolicy |
protected Timer |
rotationTimer |
protected SyncPolicy |
syncPolicy |
protected Integer |
tickTupleInterval |
protected Object |
writeLock |
Constructor and Description |
---|
AbstractHdfsBolt() |
Modifier and Type | Method and Description |
---|---|
protected abstract void |
closeOutputFile() |
protected abstract org.apache.hadoop.fs.Path |
createOutputFile() |
void |
declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) |
protected abstract void |
doPrepare(Map conf,
TopologyContext topologyContext,
OutputCollector collector) |
void |
execute(Tuple tuple) |
Map<String,Object> |
getComponentConfiguration() |
void |
prepare(Map conf,
TopologyContext topologyContext,
OutputCollector collector)
Marked as final to prevent override.
|
protected void |
rotateOutputFile() |
protected abstract void |
syncTuples()
Make the best effort to sync written data to the underlying file system.
|
protected abstract void |
writeTuple(Tuple tuple)
writes a tuple to the underlying filesystem but makes no guarantees about syncing data.
|
cleanup
protected ArrayList<RotationAction> rotationActions
protected OutputCollector collector
protected transient org.apache.hadoop.fs.FileSystem fs
protected SyncPolicy syncPolicy
protected FileRotationPolicy rotationPolicy
protected FileNameFormat fileNameFormat
protected int rotation
protected String fsUrl
protected String configKey
protected transient Object writeLock
protected transient Timer rotationTimer
protected long offset
protected Integer fileRetryCount
protected Integer tickTupleInterval
protected transient org.apache.hadoop.conf.Configuration hdfsConfig
protected void rotateOutputFile() throws IOException
IOException
public final void prepare(Map conf, TopologyContext topologyContext, OutputCollector collector)
Marked as final to prevent override. Subclasses should implement the doPrepare() method.
conf
- topologyContext
- collector
- public final void execute(Tuple tuple)
public Map<String,Object> getComponentConfiguration()
getComponentConfiguration
in interface IComponent
getComponentConfiguration
in class BaseComponent
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer)
protected abstract void writeTuple(Tuple tuple) throws IOException
writes a tuple to the underlying filesystem but makes no guarantees about syncing data.
this.offset is also updated to reflect additional data written
tuple
- IOException
protected abstract void syncTuples() throws IOException
Make the best effort to sync written data to the underlying file system. Concrete classes should very clearly state the file state that sync guarantees. For example, HdfsBolt can make a much stronger guarantee than SequenceFileBolt.
IOException
protected abstract void closeOutputFile() throws IOException
IOException
protected abstract org.apache.hadoop.fs.Path createOutputFile() throws IOException
IOException
protected abstract void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException
IOException
Copyright © 2016 The Apache Software Foundation. All Rights Reserved.