import os; import psutil; import timeit from nlp import load_dataset mem_before = psutil.Process(os.getpid()).memory_info().rss >> 20 wiki = load_dataset("wikipedia", "20200501.en", split='train') mem_after = psutil.Process(os.getpid()).memory_info().rss >> 20 print(f"RAM memory used: {(mem_after - mem_before)} MB") s = """batch_size = 1000 for i in range(0, len(wiki), batch_size): batch = wiki[i:i + batch_size] """ time = timeit.timeit(stmt=s, number=1, globals=globals()) size = wiki.dataset_size / 2**30 print(f"Iterated over the {size:.1f} GB dataset in {time:.1f} s, i.e. {size * 8/time:.1f} Gbit/s") 链接地址： https://gist.github.com/thomwolf/13ca2b2b172b2d17ac66685aa2eeba62