Python列表分块技巧与并行编程应用-CSDN博客

本文链接：https://blog.csdn.net/python_doris/article/details/102522873

本文探讨了如何将Python列表均匀分块，包括确保每块长度一致甚至使用默认值填充的方法。同时，文章提及了这些技巧在并行编程场景中的应用。对于想提升Python编程技能的程序员和程序师来说，这是一个实用的话题。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

在这里插入图片描述
每块长度几乎相同

def partition(lst, n):
    division = len(lst) / float(n)
    return [list(lst)[int(round(division * i)): int(round(division * (i + 1)))] for i in range(n)]

'''
>>> a = [i for i in range(100)]

>>> partition(a,10)

[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
 [10, 11, 12, 13, 14, 15, 16, 17, 18, 19],
 [20, 21, 22, 23, 24, 25, 26, 27, 28, 29],
 [30, 31, 32, 33, 34, 35, 36, 37, 38, 39],
 [40, 41, 42, 43, 44, 45, 46, 47, 48, 49],
 [50, 51, 52, 53, 54, 55, 56, 57, 58, 59],
 [60, 61, 62, 63, 64, 65, 66, 67, 68, 69],
 [70, 71, 72, 73, 74, 75, 76, 77, 78, 79],
 [80, 81, 82, 83, 84, 85, 86, 87, 88, 89],
 [90, 91, 92, 93, 94, 95, 96, 97, 98, 99]]

>>> partition(a,11)
 
[[0, 1, 2, 3, 4, 5, 6, 7, 8],
 [9, 10, 11, 12, 13, 14, 15, 16, 17],
 [18, 19, 20, 21, 22, 23, 24, 25, 26],
 [27, 28, 29, 30, 31, 32, 33, 34, 35],
 [36, 37, 38, 39, 40, 41, 42, 43, 44],
 [45, 46, 47, 48, 49, 50, 51, 52, 53, 54],
 [55, 56, 57, 58, 59, 60, 61, 62, 63],
 [64, 65, 66, 67, 68, 69, 70, 71, 72],
 [73, 74, 75, 76, 77, 78, 79, 80, 81],
 [82, 83, 84, 85, 86, 87, 88, 89, 90],
 [91, 92, 93, 94, 95, 96, 97, 98, 99]]

>>> partition(a,7)

[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13],
 [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28],
 [29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42],
 [43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56],
 [57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70],
 [71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85],
 [86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99]]

'''

每块长度相同，但需要用默认值填充

from six.moves import zip_longest

def grouper(n, iterable, padvalue=None):
    "grouper(3, 'abcdefg', 'x') --> ('a','b','c'), ('d','e','f'), ('g','x','x')"
    return zip_longest(*[iter(iterable)] * n, fillvalue=padvalue)

'''
>>> print(list(grouper(9,[i for i in range(100)])))

[(0, 1, 2, 3, 4, 5, 6, 7, 8),
(9, 10, 11, 12, 13, 14, 15, 16, 17), 
(18, 19, 20, 21, 22, 23, 24, 25, 26), 
(27, 28, 29, 30, 31, 32, 33, 34, 35), 
(36, 37, 38, 39, 40, 41, 42, 43, 44), 
(45, 46, 47, 48, 49, 50, 51, 52, 53),
(54, 55, 56, 57, 58, 59, 60, 61, 62),
(63, 64, 65, 66, 67, 68, 69, 70, 71),
(72, 73, 74, 75, 76, 77, 78, 79, 80), 
(81, 82, 83, 84, 85, 86, 87, 88, 89), 
(90, 91, 92, 93, 94, 95, 96, 97, 98), 
(99, None, None, None, None, None, None, None, None)]
'''

并行编程中的案例：

    with open(file_) as f:
        with ProcessPoolExecutor(max_workers=cpu_count()) as executor:
            total = 0
            for adj_chunk in executor.map(parse_func, 
            							  grouper(int(chunksize), f)):
                adjlist.extend(adj_chunk)
                total += len(adj_chunk)