HDFS的API操作

环境：maven、idea（在Windows下配置）

预备工作：一、安装jdk并配置环境变量

              二、安装idea

              三、安装maven

               1、下载maven

               2、配置maven的环境变量

             （1）设置MAVEN_HOME为maven文件夹的路径

             （2）在path中添加  “%MAVEN_HOME%\bin”

             （3）在idea中创建maven项目（注意在编写代码前应修改“pom.xml”，hdfs文件操作需要导入junit、Hadoop-hdfs以及Hadoop-common的jar包）

接下来直接上代码（注意以下操作应建立在自己hdfs服务开启的前提下实行）：

1、创建目录

import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
public class Mkdir {
    public static void main(String[] args) throws IOException, URISyntaxException {
        System.setProperty("HADOOP_USER_NAME","root");
        Configuration conf=new Configuration();
        conf.set("fs.defaultFS","hdfs://192.168.109.167:9000");                             
        FileSystem fs=FileSystem.get(conf);
        fs.mkdirs(new Path("/newFile3"));
        fs.close();
//        URI uri=new URI("hdfs://192.168.109.167:9000");
//        Configuration con=new Configuration();
//        FileSystem fs=FileSystem.get(uri,con);
//        fs.mkdirs(new Path("/newfile4"));
//        fs.close();
    }
}

2、创建文件

import java.io.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
public class Touch{
    public static void main(String[] args) throws IOException{
        System.setProperty("HADOOP_USER_NAME","root");
        Configuration conf=new Configuration();
        conf.set("fs.defaultFS","hdfs://192.168.109.167:9000");    
        FileSystem fs=FileSystem.get(conf);
        Path path=new Path("/aa");
        FSDataOutputStream out= fs.create(path,true);
        out.write("hello world!".getBytes());
        out.close();
        fs.close();
    }
}

3、读取文件内容

import java.io.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
public class Read{
    public static void main(String[] args) throws IOException{
        System.setProperty("HADOOP_USER_NAME","root");
        Configuration conf=new Configuration();
        conf.set("fs.defaultFS","hdfs://192.168.109.167:9000");    
        FileSystem fs=FileSystem.get(conf);
        Path path=new Path("/bbb");
        FSDataInputStream fis=fs.open(path);   //定义FSDataInputStream类型fis读入数据
        byte[] bytes = new byte[1024];    //声明字节数组bytes，以字节流方式读入数据
        int len = -1;
        while ((len=fis.read(bytes))!=-1){  //判断条件为fis.read(bytes)是否为-1，若为-1，即无字节可读取
            System.out.println(new String(bytes,0,len)); //字节数组转化为字符串类型，并进行打印
        }
        fs.close();
    }
}

4、文件的删除

import java.io.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
public class Delete{
    public static void main(String[] args) throws IOException{
        System.setProperty("HADOOP_USER_NAME","root");
        Configuration conf=new Configuration();
        conf.set("fs.defaultFS","hdfs://192.168.109.167:9000");    
        FileSystem fs=FileSystem.get(conf);
        Path path=new Path("/bbb");
        fs.delete(path,true);
        fs.close();
    }
}

5、文件的上传

import java.io.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
public class Put{
    public static void main(String[] args) throws IOException{
        System.setProperty("HADOOP_USER_NAME","root");
        Configuration conf=new Configuration();
        conf.set("fs.defaultFS","hdfs://192.168.109.167:9000");    
        FileSystem fs=FileSystem.get(conf);
        Path src=new Path("/yty.txt");                  //本地文件
        Path dst=new Path("/yty1.txt");                 //hdfs文件
        fs.copyFromLocalFile(src,dst);
        fs.close();
    }
}

6、文件的下载

import java.io.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
public class Download{
    public static void main(String[] args) throws IOException{
        System.setProperty("HADOOP_USER_NAME","root");
        Configuration conf=new Configuration();
        conf.set("fs.defaultFS","hdfs://192.168.109.167:9000");    
        FileSystem fs=FileSystem.get(conf);
        FSDataInputStream in=fs.open(new Path("/aa"));
        FileOutputStream out=new FileOutputStream("/aaa.txt");
        IOUtils.copyBytes(in,out,4096,true) ;
    }
}

import java.io.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
public class Getdir{
    public static void main(String[] args) throws IOException{
        System.setProperty("HADOOP_USER_NAME","root");
        Configuration conf=new Configuration();
        conf.set("fs.defaultFS","hdfs://192.168.109.167:9000");    
        FileSystem fs=FileSystem.get(conf);
        FileStatus[] fistatus=fs.listStatus(new Path("/"));
        for (FileStatus f:fistatus){
            String isdir=f.isDirectory()?"Directory":"file";
            String permission=f.getPermission().toString();
            short replication=f.getReplication();
            long len=f.getLen();
            String path=f.getPath().toString();
            System.out.println(isdir+"\t"+permission+"\t"+replication+"\t"+len+"\t"+path);
        }
    }
}

8、复制文件（hdfs上的复制）

import java.io.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
public class Copy{
    public static void main(String[] args) throws IOException{
        System.setProperty("HADOOP_USER_NAME","root");
        Configuration conf=new Configuration();
        conf.set("fs.defaultFS","hdfs://192.168.109.167:9000");    
        FileSystem fs=FileSystem.get(conf);
        fs.create(new Path("/bbb"));
        Path newpath=new Path("/bbb");
        FSDataOutputStream out= fs.create(newpath);
        Path oldpath=new Path("/aaa");
        FSDataInputStream fis=fs.open(oldpath);   //定义FSDataInputStream类型fis读入数据
        byte[] bytes = new byte[1024];    //声明字节数组bytes，以字节流方式读入数据
        int len = -1;
        while ((len=fis.read(bytes))!=-1){  //判断条件为fis.read(bytes)是否为-1，若为-1，即无字节可读取
            out.write(bytes,0,len);
        }
        out.close();
    }
}

9、重命名文件

import java.io.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
public class Rename{
    public static void main(String[] args) throws IOException{
        System.setProperty("HADOOP_USER_NAME","root");
        Configuration conf=new Configuration();
        conf.set("fs.defaultFS","hdfs://192.168.109.167:9000");    
        FileSystem fs=FileSystem.get(conf);
        Path old=new Path("/aa");
        Path now=new Path("/bb");
        fs.rename(old,now);
    }
}

之后可将代码打包至hdfs运行，命令格式如hadoop jar /jar包所在位置包名.类名（若是先前pom.xml配置好main所在位置，此处应该不用指定包名）

标签： maven idea hadoop

本文转载自: https://blog.csdn.net/m0_63716485/article/details/127537422
版权归原作者 m0_63716485 所有，如有侵权，请联系我们删除。

发表评论

“HDFS的API操作”的评论:

关于作者

overfit同步小助手

相关阅读

文章导航