简单的Http请求数据保存到Hdfs

package com.etl;import java.io.IOException;import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;import okhttp3.Call;
import okhttp3.Callback;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;public class LinkHttp { private static Configuration conf = null;
private static String fsName = "fs.defaultFS";
private static String fsURI = null; public static void main(String[] args) throws Exception { String name = args[0];
String uri = args[1];
String url = args[2];
String targetFile = args[3]; //文件全路径 //初始化
init(name, uri);
OkHttpClient client = new OkHttpClient();
final Request request = new Request.Builder().url(url).get().build();
Call call = client.newCall(request);
call.enqueue(new Callback() { @Override
public void onFailure(Call call, IOException e) {
System.out.println("Fail");
} @Override
public void onResponse(Call call, Response response) throws IOException {
FileSystem fs = null;
try {
Path dstPath = new Path(targetFile);
fs = FileSystem.get(conf);
FSDataOutputStream outputStream = fs.create(dstPath);
if(response.isSuccessful()) {
// System.out.println(response.body().string());
outputStream.write(response.body().bytes());
outputStream.close();
System.out.println("create file " + targetFile + " success!");
//fs.close();
}
}catch (Exception e){
e.printStackTrace();
}finally {
fs.close();
}
System.out.println("run writeHdfs end"); //关闭
if(response.body()!=null) {
response.body().close();
}
}
}); } private static void init(String name, String uri) {
if(StringUtils.isNotBlank(fsName)){
fsName = name;
}
fsURI = uri;
conf = new Configuration();
conf.set(fsName, fsURI);
}}

#!/bin/sh
name=fs.defaultFS #固定不变
uri=dwpro-name1:8020 #hdfs文件系统地址
url=http://www.cnblogs.com/30go/ #待保存的http地址
targetPath=/tmp/test/king.txt # 目标的文件名
java -Djava.ext.dirs=lib com.etl.LinkHttp \
${name} ${uri} ${url} ${targetPath} >> test.log >& &