2021-09-23

统计HDFS元数据

编写offline_fsimage.sh脚本，解析fsimage文件获取HDFS元数据并上传至HDFS。

#!/bin/bash

# 加载基本配置信息
# source ./config-env.sh

# 集群启用了Kerberos，先使用hdfs的Kerberos账号
# kinit -kt hdfs.keytab hdfs/admin

# 将HDFS的FsImage导出
mkdir -p ./tmp_meta
rm -rf ./tmp_meta/*
hdfs dfsadmin -fetchImage ./tmp_meta

if [ $? -ne 0 ];then
	echo "获取FsImage失败..."
	exit
fi

# 使用hdfs提供的oiv解析FsImage数据文件，将FsImage转换为csv格式数据
hdfs oiv -i ./tmp_meta/* -o ./tmp_meta/fsimage.csv -p Delimited

# 将生成的csv文件头去掉，并上传至HDFS的/tmp目录
sed -i -e "1d" ./tmp_meta/fsimage.csv

# 切换为Hive的用户进行put
# kinit -kt hive.keytab hive/admin

TMP_DIR=/tmp/meta/fsimage
hdfs dfs -rmr $TMP_DIR
hdfs dfs -mkdir -p $TMP_DIR
echo "创建HDFS临时目录[${TMP_DIR}]"
hdfs dfs -copyFromLocal ./tmp_meta/fsimage.csv ${TMP_DIR}

if [ $? -ne 0 ];then
	echo "上传fsimage.csv文件失败..."
	exit
fi

echo "上传[./tmp_meta/fsimage.csv文件至[${TMP_DIR}]成功"

创建HDFS元数据表。

create database if not exists meta;
use meta;

-- 创建FsImage临时表
CREATE EXTERNAL TABLE if not exists hdfs_meta_tmp (
  path string,
  repl int,
  modification_time string,
  access_time string,
  preferred_block_size int,
  block_count double,
  file_size double,
  ns_quota int,
  ds_quota int,
  permission string,
  username string,
  groupname string
  )
PARTITIONED BY (
  par_dt string)
ROW FORMAT SERDE
  'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
  'field.delim'='\t',
  'serialization.format'='\t')
STORED AS INPUTFORMAT
  'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT
  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION '/tmp/meta/fsimage';