Win a copy of The Little Book of Impediments (e-book only) this week in the Agile and Other Processes forum!
  • Post Reply
  • Bookmark Topic Watch Topic
  • New Topic

FileInputStream not saving the .seq file after finishing Hadoop

 
hussein haggag
Greenhorn
Posts: 1
  • Mark post as helpful
  • send pies
  • Quote
  • Report post to moderator
I'm using Hadoop on my Ubuntu machine. And I'm trying to write a seq file using the below code, but the code finishes and says "sucessfully (or) read 202000 ...." and on the destintation folder I can't find my output seq file. So if anyone could please advise.

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.FileInputStream;
import java.io.BufferedInputStream;
import java.io.DataInputStream;

import java.nio.ByteBuffer;

import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.GenericOptionsParser;

import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class MainUpload_sp
extends Configured
implements Tool
{


public static void main(String[] args) throws Exception {
int res = ToolRunner.run(new Configuration(), new MainUpload_sp(), args);
System.exit(res);
}

public int run(String[] args) throws Exception {

System.out.println("UPLOADING Program");


if( args.length != 2 ) {
System.err.println("usage: hadoop xxx.jar upload_sp <localfilein.dat> <hdfsfileout.seq>");
return -1;
}
String filename_in = args[0];
String filename_out = args[1];

Configuration conf = getConf();


FileInputStream fin = new FileInputStream(filename_in);
BufferedInputStream bfin = new BufferedInputStream(fin, 8192);


conf.set("fs.file.impl","org.apache.hadoop.fs.LocalFileSystem");
FileSystem fs = FileSystem.get(conf);
if( fs.getConf() == null) {
System.err.println("Could not initialize the local filesystem");
return -1;
}


java.io.File fileout = new java.io.File(filename_out);

Path path_out = new Path(fileout.getPath());
SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf,
path_out,
LongWritable.class,
LabeledSplitPointBuff.class,
SequenceFile.CompressionType.NONE);

long numRecords;

try {
numRecords = 0;

LongWritable Key = new LongWritable(0);
LabeledSplitPointBuff lsp = new LabeledSplitPointBuff();
while(true) {

int read = bfin.read(lsp.data, 0, LabeledSplitPointBuff.RECORDSIZE);
System.out.println("read "+read);
if (read != LabeledSplitPointBuff.RECORDSIZE) break;
writer.append(Key, lsp);
numRecords += 1;
}
} finally {
writer.close();
}

System.out.println("sucessfully (or) read " + Long.toString(numRecords)
+ " from " + filename_in + " to " + filename_out);


return 0;
}

}
 
Rohini Sahuji
Greenhorn
Posts: 28
1
Chrome Java
  • Mark post as helpful
  • send pies
  • Quote
  • Report post to moderator
I guess you are creating destination file in local file system.. and not in hdfs...
 
  • Post Reply
  • Bookmark Topic Watch Topic
  • New Topic