如何按数量百分比分割文件。线？
假设我想将文件分成 3 部分（60%/20%/20% 部分），我可以手动执行此操作，-_- :
$ wc -l brown.txt
57339 brown.txt
$ bc <<< "57339 / 10 * 6"
34398
$ bc <<< "57339 / 10 * 2"
11466
$ bc <<< "34398 + 11466"
45864
bc <<< "34398 + 11466 + 11475"
57339
$ head -n 34398 brown.txt > part1.txt
$ sed -n 34399,45864p brown.txt > part2.txt
$ sed -n 45865,57339p brown.txt > part3.txt
$ wc -l part*.txt
34398 part1.txt
11466 part2.txt
11475 part3.txt
57339 total
但我确信有更好的方法！
有一个实用程序将行号作为参数，该行号应成为每个新文件的第一个：
csplit#!/bin/bash
usage () {
printf '%s\n' "${0##*/} [-ks] [-f prefix] [-n number] file arg1..." >&2
}
# Collect csplit options
while getopts "ksf:n:" opt; do
case "$opt" in
k|s) args+=(-"$opt") ;; # k: no remove on error, s: silent
f|n) args+=(-"$opt" "$OPTARG") ;; # f: filename prefix, n: digits in number
*) usage; exit 1 ;;
esac
done
shift $(( OPTIND - 1 ))
fname=$1
shift
ratios=("$@")
len=$(wc -l < "$fname")
# Sum of ratios and array of cumulative ratios
for ratio in "${ratios[@]}"; do
(( total += ratio ))
cumsums+=("$total")
done
# Don't need the last element
unset cumsums[-1]
# Array of numbers of first line in each split file
for sum in "${cumsums[@]}"; do
linenums+=( $(( sum * len / total + 1 )) )
done
csplit "${args[@]}" "$fname" "${linenums[@]}"
在要分割的文件的名称之后，它采用分割文件的大小相对于其总和的比率，即
percsplit brown.txt 60 20 20
percsplit brown.txt 6 2 2
percsplit brown.txt 3 1 1
都是等价的。
与问题中的情况类似的用法如下：
$ percsplit -s -f part -n 1 brown.txt 60 20 20
$ wc -l part*
34403 part0
11468 part1
11468 part2
57339 total
编号从零开始，并且没有
txt--suffix-format.txtgetopts此解决方案适用于非常短的文件（将两行文件分成两部分），并且繁重的工作由
csplit$ cat file
a
b
c
d
e
$ cat tst.awk
BEGIN {
split(pcts,p)
nrs[1]
for (i=1; i in p; i++) {
pct += p[i]
nrs[int(size * pct / 100) + 1]
}
}
NR in nrs{ close(out); out = "part" ++fileNr ".txt" }
{ print $0 " > " out }
$ awk -v size=$(wc -l < file) -v pcts="60 20 20" -f tst.awk file
a > part1.txt
b > part1.txt
c > part1.txt
d > part2.txt
e > part3.txt
将
" > ">以下 bash 脚本允许您指定百分比，例如
./split.sh brown.txt 60 20 20
您还可以使用占位符
../split.sh brown.txt 60 20 .
分割后的文件被写入
part1-brown.txt
part2-brown.txt
part3-brown.txt
脚本始终生成与指定数量一样多的
partcat part*#! /bin/bash
file="$1"
fileLength=$(wc -l < "$file")
shift
part=1
percentSum=0
currentLine=1
for percent in "$@"; do
[ "$percent" == "." ] && ((percent = 100 - percentSum))
((percentSum += percent))
if ((percent < 0 || percentSum > 100)); then
echo "invalid percentage" 1>&2
exit 1
fi
((nextLine = fileLength * percentSum / 100))
if ((nextLine < currentLine)); then
printf "" # create empty file
else
sed -n "$currentLine,$nextLine"p "$file"
fi > "part$part-$file"
((currentLine = nextLine + 1))
((part++))
done
BEGIN {
split(w, weight)
total = 0
for (i in weight) {
weight[i] += total
total = weight[i]
}
}
FNR == 1 {
if (NR!=1) {
write_partitioned_files(weight,a)
split("",a,":") #empty a portably
}
name=FILENAME
}
{a[FNR]=$0}
END {
write_partitioned_files(weight,a)
}
function write_partitioned_files(weight, a) {
split("",threshold,":")
size = length(a)
for (i in weight){
threshold[length(threshold)] = int((size * weight[i] / total)+0.5)+1
}
l=1
part=0
for (i in threshold) {
close(out)
out = name ".part" ++part
for (;l<threshold[i];l++) {
print a[l] " > " out
}
}
}
调用为：
awk -v w="60 20 20" -f above_script.awk file_to_split1 file_to_split2 ...
将脚本中的
" > ">变量
w"2 1 1 3"对于大文件，数组
aawkBEGIN {
split(w, weight)
for (i in weight) {
total += weight[i]; weight[i] = total #cumulative sum
}
}
FNR == 1 {
#get number of lines. take care of single quotes in filename.
name = gensub("'", "'\"'\"'", "g", FILENAME)
"wc -l '" name "'" | getline size
split("", threshold, ":")
for (i in weight){
threshold[length(threshold)+1] = int((size * weight[i] / total)+0.5)+1
}
part=1; close(out); out = FILENAME ".part" part
}
{
if(FNR>=threshold[part]) {
close(out); out = FILENAME ".part" ++part
}
print $0 " > " out
}
这会遍历每个文件两次。一次用于计算行数（通过
wc -l我喜欢Benjamin W.的
csplit#!/bin/bash
# usage ./splitpercs.sh file 60 20 20
n=`wc -l <"$1"` || exit 1
echo $* | tr ' ' '\n' | tail -n+2 | head -n`expr $# - 1` |
awk -v n=$n 'BEGIN{r=1} {r+=n*$0/100; if(r > 1 && r < n){printf "%d\n",r}}' |
uniq | xargs csplit -sfpart "$1"
（
if(r > 1 && r < n)uniq我只是按照您的指导，将您手动执行的操作写入脚本中。 它可能不是最快或“最好”的，但如果您了解自己现在在做什么并且可以将其“脚本化”，那么如果您需要维护它，情况可能会更好。
#!/bin/bash
# thisScript.sh yourfile.txt 20 50 10 20
YOURFILE=$1
shift
# changed to cat | wc so I dont have to remove the filename which comes from
# wc -l
LINES=$(cat $YOURFILE | wc -l )
startpct=0;
PART=1;
for pct in $@
do
# I am assuming that each parameter is on top of the last
# so 10 30 10 would become 10, 10+30 = 40, 10+30+10 = 50, ...
endpct=$( echo "$startpct + $pct" | bc)
# your math but changed parts of 100 instead of parts of 10.
# change bc <<< to echo "..." | bc
# so that one can capture the output into a bash variable.
FIRSTLINE=$( echo "$LINES * $startpct / 100 + 1" | bc )
LASTLINE=$( echo "$LINES * $endpct / 100" | bc )
# use sed every time because the special case for head
# doesn't really help performance.
sed -n $FIRSTLINE,${LASTLINE}p $YOURFILE > part${PART}.txt
$((PART++))
startpct=$endpct
done
# get the rest if the % dont add to 100%
if [[ $( "lastpct < 100" | bc ) -gt 0 ]] ; then
sed -n $FIRSTLINE,${LASTLINE}p $YOURFILE > part${PART}.txt
fi
wc -l part*.txt
其实有一个简单的方法：
vim -c "exe 'norm 20%' | 1,. w! >> 1_tmp_01.csv | q!" 1_db.csv
vim -c "exe 'norm 20%' | exe 'norm j' | exe 'norm ma' | exe 'norm 40%' | exe 'norm mb' | 'a,'b w! >> 1_tmp_02.csv | q!" 1_db.csv
vim -c "exe 'norm 40%' | exe 'norm j' | exe 'norm ma' | exe 'norm 60%' | exe 'norm mb' | 'a,'b w! >> 1_tmp_03.csv | q!" 1_db.csv
vim -c "exe 'norm 60%' | exe 'norm j' | exe 'norm ma' | exe 'norm 80%' | exe 'norm mb' | 'a,'b w! >> 1_tmp_04.csv | q!" 1_db.csv
vim -c "exe 'norm 80%' | exe 'norm j' | .,$ w! >> 1_tmp_05.csv | q!" 1_db.csv.
我不确定 vim 作为一个工具是否会受到威胁，但对我来说它可以完成这项工作。