如何使用python以相反的顺序读取文件？我想从最后一行到第一行读取一个文件。
for line in reversed(open("filename").readlines()):
print line.rstrip()
在Python 3中：
for line in reversed(list(open("filename"))):
print(line.rstrip())
如果您担心文件大小/内存使用情况，内存映射文件和向后扫描换行是一种解决方案：
对于具有不适合内存的大文件的情况，接受的答案将不起作用（这种情况并不罕见）。
正如其他人所指出的那样，@srohde answer看起来不错，但它有下一个问题：
utf-8编码的文件和非ascii内容
й
通过buf_size等于1并将有
UnicodeDecodeError: 'utf8' codec can't decode byte 0xb9 in position 0: invalid start byte
当然文本可能会更大，但buf_size可能会被拾取，所以它会导致像上面那样的混淆错误，所以考虑到所有这些问题，我写了不同的功能：
首先让我们定义下一个效用函数：
ceil_division用天花板进行划分（与标准的//划分与地板相比，更多信息可以在this thread找到）
def ceil_division(left_number, right_number):
"""
Divides given numbers with ceiling.
"""
return -(-left_number // right_number)
split用于通过给定分隔符从右端分割字符串并具有保留它的能力：
def split(string, separator, keep_separator):
"""
Splits given string by given separator.
"""
parts = string.split(separator)
if keep_separator:
*parts, last_part = parts
parts = [part + separator for part in parts]
if last_part:
return parts + [last_part]
return parts
read_batch_from_end从二进制流的右端读取批处理
def read_batch_from_end(byte_stream, size, end_position):
"""
Reads batch from the end of given byte stream.
"""
if end_position > size:
offset = end_position - size
else:
offset = 0
size = end_position
byte_stream.seek(offset)
return byte_stream.read(size)
之后我们可以定义以相反顺序读取字节流的函数
import functools
import itertools
import os
from operator import methodcaller, sub
def reverse_binary_stream(byte_stream, batch_size=None,
lines_separator=None,
keep_lines_separator=True):
if lines_separator is None:
lines_separator = (b'\r', b'\n', b'\r\n')
lines_splitter = methodcaller(str.splitlines.__name__,
keep_lines_separator)
else:
lines_splitter = functools.partial(split,
separator=lines_separator,
keep_separator=keep_lines_separator)
stream_size = object_.seek(0, os.SEEK_END)
if batch_size is None:
batch_size = stream_size or 1
batches_count = ceil_division(stream_size, batch_size)
remaining_bytes_indicator = itertools.islice(
itertools.accumulate(itertools.chain([stream_size],
itertools.repeat(batch_size)),
sub),
batches_count)
try:
remaining_bytes_count = next(remaining_bytes_indicator)
except StopIteration:
return
def read_batch(position: int) -> bytes:
result = read_batch_from_end(object_,
size=batch_size,
end_position=position)
while result.startswith(lines_separator):
try:
position = next(remaining_bytes_indicator)
except StopIteration:
break
result = (read_batch_from_end(object_,
size=batch_size,
end_position=position)
+ result)
return result
batch = read_batch(remaining_bytes_count)
segment, *lines = lines_splitter(batch)
yield from reverse(lines)
for remaining_bytes_count in remaining_bytes_indicator:
batch = read_batch(remaining_bytes_count)
lines = lines_splitter(batch)
if batch.endswith(lines_separator):
yield segment
else:
lines[-1] += segment
segment, *lines = lines
yield from reverse(lines)
yield segment
最后一个用于反转文本文件的函数可以定义如下：
import codecs
def reverse_file(file, batch_size=None,
lines_separator=None,
keep_lines_separator=True):
encoding = file.encoding
if lines_separator is not None:
lines_separator = lines_separator.encode(encoding)
yield from map(functools.partial(codecs.decode,
encoding=encoding),
reverse_binary_stream(
file.buffer,
batch_size=batch_size,
lines_separator=lines_separator,
keep_lines_separator=keep_lines_separator))
我使用fsutil command生成了4个文件：
我也重构了@srohde解决方案来处理文件对象而不是文件路径。
from timeit import Timer
repeats_count = 7
number = 1
create_setup = ('from collections import deque\n'
'from __main__ import reverse_file, reverse_readline\n'
'file = open("{}")').format
srohde_solution = ('with file:\n'
' deque(reverse_readline(file,\n'
' buf_size=8192),'
' maxlen=0)')
azat_ibrakov_solution = ('with file:\n'
' deque(reverse_file(file,\n'
' lines_separator="\\n",\n'
' keep_lines_separator=False,\n'
' batch_size=8192), maxlen=0)')
print('reversing empty file by "srohde"',
min(Timer(srohde_solution,
create_setup('empty.txt')).repeat(repeats_count, number)))
print('reversing empty file by "Azat Ibrakov"',
min(Timer(azat_ibrakov_solution,
create_setup('empty.txt')).repeat(repeats_count, number)))
print('reversing tiny file (1MB) by "srohde"',
min(Timer(srohde_solution,
create_setup('tiny.txt')).repeat(repeats_count, number)))
print('reversing tiny file (1MB) by "Azat Ibrakov"',
min(Timer(azat_ibrakov_solution,
create_setup('tiny.txt')).repeat(repeats_count, number)))
print('reversing small file (10MB) by "srohde"',
min(Timer(srohde_solution,
create_setup('small.txt')).repeat(repeats_count, number)))
print('reversing small file (10MB) by "Azat Ibrakov"',
min(Timer(azat_ibrakov_solution,
create_setup('small.txt')).repeat(repeats_count, number)))
print('reversing large file (50MB) by "srohde"',
min(Timer(srohde_solution,
create_setup('large.txt')).repeat(repeats_count, number)))
print('reversing large file (50MB) by "Azat Ibrakov"',
min(Timer(azat_ibrakov_solution,
create_setup('large.txt')).repeat(repeats_count, number)))
注意：我使用collections.deque类来排气发电机。
对于Windows 10上的PyPy 3.5：
reversing empty file by "srohde" 8.31e-05
reversing empty file by "Azat Ibrakov" 0.00016090000000000028
reversing tiny file (1MB) by "srohde" 0.160081
reversing tiny file (1MB) by "Azat Ibrakov" 0.09594989999999998
reversing small file (10MB) by "srohde" 8.8891863
reversing small file (10MB) by "Azat Ibrakov" 5.323388100000001
reversing large file (50MB) by "srohde" 186.5338368
reversing large file (50MB) by "Azat Ibrakov" 99.07450229999998
对于Windows 10上的Python 3.5：
reversing empty file by "srohde" 3.600000000000001e-05
reversing empty file by "Azat Ibrakov" 4.519999999999958e-05
reversing tiny file (1MB) by "srohde" 0.01965560000000001
reversing tiny file (1MB) by "Azat Ibrakov" 0.019207699999999994
reversing small file (10MB) by "srohde" 3.1341862999999996
reversing small file (10MB) by "Azat Ibrakov" 3.0872588000000007
reversing large file (50MB) by "srohde" 82.01206720000002
reversing large file (50MB) by "Azat Ibrakov" 82.16775059999998
因此，我们可以看到它表现得像原始解决方案，但更为通用，没有上面列出的缺点。
我已将此添加到0.3.0版本的lz package（需要Python 3.5+），它有许多经过良好测试的功能/迭代实用程序。
可以像
import io
from lz.iterating import reverse
...
with open('path/to/file') as file:
for line in reverse(file, batch_size=io.DEFAULT_BUFFER_SIZE):
print(line)
它支持all standard encodings（可能除了utf-7，因为我很难定义a strategy来生成可编码的字符串）。
def reverse_lines(filename):
y=open(filename).readlines()
return y[::-1]
在处理文件时始终使用with，因为它会为您处理所有事情：
with open('filename', 'r') as f:
for line in reversed(f.readlines()):
print line
或者在Python 3中：
with open('filename', 'r') as f:
for line in reversed(list(f.readlines())):
print(line)
您需要先以读取格式打开文件，将其保存到变量中，然后以写入格式打开第二个文件，您可以使用[:: - 1]切片编写或附加变量，从而完全反转文件。您还可以使用readlines（）将其转换为可以操作的行列表
def copy_and_reverse(filename, newfile):
with open(filename) as file:
text = file.read()
with open(newfile, "w") as file2:
file2.write(text[::-1])
在做任何事情之前，大多数答案都需要读取整个文件。该样本从最后读取越来越大的样本。
在写这个答案的时候，我只看到了MuratYükselen的回答。它几乎相同，我认为这是一件好事。下面的示例还处理\ r并在每一步增加其缓冲区大小。我也有一些unit tests支持这个代码。
def readlines_reversed(f):
""" Iterate over the lines in a file in reverse. The file must be
open in 'rb' mode. Yields the lines unencoded (as bytes), including the
newline character. Produces the same result as readlines, but reversed.
If this is used to reverse the line in a file twice, the result is
exactly the same.
"""
head = b""
f.seek(0, 2)
t = f.tell()
buffersize, maxbuffersize = 64, 4096
while True:
if t <= 0:
break
# Read next block
buffersize = min(buffersize * 2, maxbuffersize)
tprev = t
t = max(0, t - buffersize)
f.seek(t)
lines = f.read(tprev - t).splitlines(True)
# Align to line breaks
if not lines[-1].endswith((b"\n", b"\r")):
lines[-1] += head # current tail is previous head
elif head == b"\n" and lines[-1].endswith(b"\r"):
lines[-1] += head # Keep \r\n together
elif head:
lines.append(head)
head = lines.pop(0) # can be '\n' (ok)
# Iterate over current block in reverse
for line in reversed(lines):
yield line
if head:
yield head
逐行读取文件，然后以相反的顺序将其添加到列表中。
这是一个代码示例：
reverse = []
with open("file.txt", "r") as file:
for line in file:
line = line.strip()
reverse[0:0] = line
import sys
f = open(sys.argv[1] , 'r')
for line in f.readlines()[::-1]:
print line
我不得不在前一段时间做这个并使用下面的代码。它管到壳。恐怕我不再拥有完整的剧本了。如果您使用的是unixish操作系统，则可以使用“tac”，但是例如Mac OSX tac命令不起作用，使用tail -r。下面的代码片段测试您所在的平台，并相应地调整命令
# We need a command to reverse the line order of the file. On Linux this
# is 'tac', on OSX it is 'tail -r'
# 'tac' is not supported on osx, 'tail -r' is not supported on linux.
if sys.platform == "darwin":
command += "|tail -r"
elif sys.platform == "linux2":
command += "|tac"
else:
raise EnvironmentError('Platform %s not supported' % sys.platform)
作为生成器编写的正确，有效的答案。
import os
def reverse_readline(filename, buf_size=8192):
"""A generator that returns the lines of a file in reverse order"""
with open(filename) as fh:
segment = None
offset = 0
fh.seek(0, os.SEEK_END)
file_size = remaining_size = fh.tell()
while remaining_size > 0:
offset = min(file_size, offset + buf_size)
fh.seek(file_size - offset)
buffer = fh.read(min(remaining_size, buf_size))
remaining_size -= buf_size
lines = buffer.split('\n')
# The first line of the buffer is probably not a complete line so
# we'll save it and append it to the last line of the next buffer
# we read
if segment is not None:
# If the previous chunk starts right from the beginning of line
# do not concat the segment to the last line of new chunk.
# Instead, yield the segment first
if buffer[-1] != '\n':
lines[-1] += segment
else:
yield segment
segment = lines[0]
for index in range(len(lines) - 1, 0, -1):
if lines[index]:
yield lines[index]
# Don't yield None if the file was empty
if segment is not None:
yield segment
这样的事情怎么样：
import os
def readlines_reverse(filename):
with open(filename) as qfile:
qfile.seek(0, os.SEEK_END)
position = qfile.tell()
line = ''
while position >= 0:
qfile.seek(position)
next_char = qfile.read(1)
if next_char == "\n":
yield line[::-1]
line = ''
else:
line += next_char
position -= 1
yield line[::-1]
if __name__ == '__main__':
for qline in readlines_reverse(raw_input()):
print qline
由于文件是按相反的顺序逐字读取的，因此只要单个行适合内存，它甚至可以在非常大的文件上工作。
你也可以使用python模块file_read_backwards。
安装后，通过pip install file_read_backwards（v1.2.1），您可以通过以下内容高效的方式向后（按行）读取整个文件：
#!/usr/bin/env python2.7
from file_read_backwards import FileReadBackwards
with FileReadBackwards("/path/to/file", encoding="utf-8") as frb:
for l in frb:
print l
它支持“utf-8”，“latin-1”和“ascii”编码。
python3也支持。更多文档可以在http://file-read-backwards.readthedocs.io/en/latest/readme.html找到
import re
def filerev(somefile, buffer=0x20000):
somefile.seek(0, os.SEEK_END)
size = somefile.tell()
lines = ['']
rem = size % buffer
pos = max(0, (size // buffer - 1) * buffer)
while pos >= 0:
somefile.seek(pos, os.SEEK_SET)
data = somefile.read(rem + buffer) + lines[0]
rem = 0
lines = re.findall('[^\n]*\n?', data)
ix = len(lines) - 2
while ix > 0:
yield lines[ix]
ix -= 1
pos -= buffer
else:
yield lines[0]
with open(sys.argv[1], 'r') as f:
for line in filerev(f):
sys.stdout.write(line)
在这里你可以找到我的实现，你可以通过更改“缓冲区”变量来限制ram的使用，有一个错误，程序在开头打印一个空行。
如果没有超过缓冲区字节的新行，ram的使用可能会增加，“leak”变量将增加，直到看到一个新行（“\ n”）。
这也适用于16 GB文件，这比我的总内存大。
import os,sys
buffer = 1024*1024 # 1MB
f = open(sys.argv[1])
f.seek(0, os.SEEK_END)
filesize = f.tell()
division, remainder = divmod(filesize, buffer)
line_leak=''
for chunk_counter in range(1,division + 2):
if division - chunk_counter < 0:
f.seek(0, os.SEEK_SET)
chunk = f.read(remainder)
elif division - chunk_counter >= 0:
f.seek(-(buffer*chunk_counter), os.SEEK_END)
chunk = f.read(buffer)
chunk_lines_reversed = list(reversed(chunk.split('\n')))
if line_leak: # add line_leak from previous chunk to beginning
chunk_lines_reversed[0] += line_leak
# after reversed, save the leakedline for next chunk iteration
line_leak = chunk_lines_reversed.pop()
if chunk_lines_reversed:
print "\n".join(chunk_lines_reversed)
# print the last leaked line
if division - chunk_counter < 0:
print line_leak
谢谢@srohde的答案。它有一个小错误检查带有'is'运算符的换行符，我无法评论1声誉的答案。此外，我想管理外面的文件，因为这使我能够将我的rambling嵌入到luigi任务中。
我需要改变的形式是：
with open(filename) as fp:
for line in fp:
#print line, # contains new line
print '>{}<'.format(line)
我很乐意改为：
with open(filename) as fp:
for line in reversed_fp_iter(fp, 4):
#print line, # contains new line
print '>{}<'.format(line)
这是一个修改后的答案，需要文件句柄并保留换行符：
def reversed_fp_iter(fp, buf_size=8192):
"""a generator that returns the lines of a file in reverse order
ref: https://stackoverflow.com/a/23646049/8776239
"""
segment = None # holds possible incomplete segment at the beginning of the buffer
offset = 0
fp.seek(0, os.SEEK_END)
file_size = remaining_size = fp.tell()
while remaining_size > 0:
offset = min(file_size, offset + buf_size)
fp.seek(file_size - offset)
buffer = fp.read(min(remaining_size, buf_size))
remaining_size -= buf_size
lines = buffer.splitlines(True)
# the first line of the buffer is probably not a complete line so
# we'll save it and append it to the last line of the next buffer
# we read
if segment is not None:
# if the previous chunk starts right from the beginning of line
# do not concat the segment to the last line of new chunk
# instead, yield the segment first
if buffer[-1] == '\n':
#print 'buffer ends with newline'
yield segment
else:
lines[-1] += segment
#print 'enlarged last line to >{}<, len {}'.format(lines[-1], len(lines))
segment = lines[0]
for index in range(len(lines) - 1, 0, -1):
if len(lines[index]):
yield lines[index]
# Don't yield None if the file was empty
if segment is not None:
yield segment
一个简单的函数来创建第二个文件reverse（仅限linux）：
import os
def tac(file1, file2):
print(os.system('tac %s > %s' % (file1,file2)))
如何使用
tac('ordered.csv', 'reversed.csv')
f = open('reversed.csv')