GDAL モジュール ( https://pypi.org/project/GDAL/ )を使用して大きなラスター イメージを開き、渡された座標タプルのリストで定義されたいくつかの場所で小さなイメージを抽出するクラスがあります。私のコードのこの単純化されたバージョンのように、Pathos を使用して座標の大きなリストを処理したい:
import gdal
import pathos.pools as pp
class MyClass:
def __init__(self, image):
self.image_object = gdal.Open(image)
def get_small_image(self, coord, size=100):
small_image = self.image_object.ReadAsArray(coord[0], coord[1], size, size)
return small_image
def run_multi(self, coords_in):
pool = pp.ProcessPool(2)
output = pool.map(self.get_small_image, coords_in)
return output
image = r'C:\path\to\image.JP2'
class_obj = MyClass(image)
coords = [(200, 200), (400, 400), (600, 600), (800, 800)]
results = class_obj.run_multi(coords)
print(results)
これを実行すると、次のエラーが発生します。
Traceback (most recent call last):
File "C:/Users/lharris/PycharmProjects/road_condition_py3/error_recreate.py", line 26, in <module>
results = class_obj.run_multi(coords)
File "C:/Users/lharris/PycharmProjects/road_condition_py3/error_recreate.py", line 17, in run_multi
output = pool.map(self.get_small_image, coords_in)
File "C:\Users\lharris\PycharmProjects\road_condition_py3\venv\lib\site-packages\pathos\multiprocessing.py", line 137, in map
return _pool.map(star(f), zip(*args)) # chunksize
File "C:\Users\lharris\PycharmProjects\road_condition_py3\venv\lib\site-packages\multiprocess\pool.py", line 266, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "C:\Users\lharris\PycharmProjects\road_condition_py3\venv\lib\site-packages\multiprocess\pool.py", line 644, in get
raise self._value
File "C:\Users\lharris\PycharmProjects\road_condition_py3\venv\lib\site-packages\multiprocess\pool.py", line 424, in _handle_tasks
put(task)
File "C:\Users\lharris\PycharmProjects\road_condition_py3\venv\lib\site-packages\multiprocess\connection.py", line 209, in send
self._send_bytes(_ForkingPickler.dumps(obj))
File "C:\Users\lharris\PycharmProjects\road_condition_py3\venv\lib\site-packages\multiprocess\reduction.py", line 54, in dumps
cls(buf, protocol).dump(obj)
File "C:\Python36\lib\pickle.py", line 409, in dump
self.save(obj)
File "C:\Python36\lib\pickle.py", line 476, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Python36\lib\pickle.py", line 751, in save_tuple
save(element)
File "C:\Python36\lib\pickle.py", line 476, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Python36\lib\pickle.py", line 736, in save_tuple
save(element)
File "C:\Python36\lib\pickle.py", line 476, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Python36\lib\pickle.py", line 736, in save_tuple
save(element)
File "C:\Python36\lib\pickle.py", line 476, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Users\lharris\PycharmProjects\road_condition_py3\venv\lib\site-packages\dill\_dill.py", line 1377, in save_function
obj.__dict__), obj=obj)
File "C:\Python36\lib\pickle.py", line 610, in save_reduce
save(args)
File "C:\Python36\lib\pickle.py", line 476, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Python36\lib\pickle.py", line 751, in save_tuple
save(element)
File "C:\Python36\lib\pickle.py", line 476, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Python36\lib\pickle.py", line 736, in save_tuple
save(element)
File "C:\Python36\lib\pickle.py", line 476, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Users\lharris\PycharmProjects\road_condition_py3\venv\lib\site-packages\dill\_dill.py", line 1120, in save_cell
pickler.save_reduce(_create_cell, (f,), obj=obj)
File "C:\Python36\lib\pickle.py", line 610, in save_reduce
save(args)
File "C:\Python36\lib\pickle.py", line 476, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Python36\lib\pickle.py", line 736, in save_tuple
save(element)
File "C:\Python36\lib\pickle.py", line 476, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Users\lharris\PycharmProjects\road_condition_py3\venv\lib\site-packages\dill\_dill.py", line 1069, in save_instancemethod0
pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj)
File "C:\Python36\lib\pickle.py", line 610, in save_reduce
save(args)
File "C:\Python36\lib\pickle.py", line 476, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Python36\lib\pickle.py", line 736, in save_tuple
save(element)
File "C:\Python36\lib\pickle.py", line 521, in save
self.save_reduce(obj=obj, *rv)
File "C:\Python36\lib\pickle.py", line 634, in save_reduce
save(state)
File "C:\Python36\lib\pickle.py", line 476, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Users\lharris\PycharmProjects\road_condition_py3\venv\lib\site-packages\dill\_dill.py", line 893, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "C:\Python36\lib\pickle.py", line 821, in save_dict
self._batch_setitems(obj.items())
File "C:\Python36\lib\pickle.py", line 852, in _batch_setitems
save(v)
File "C:\Python36\lib\pickle.py", line 521, in save
self.save_reduce(obj=obj, *rv)
File "C:\Python36\lib\pickle.py", line 634, in save_reduce
save(state)
File "C:\Python36\lib\pickle.py", line 476, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Users\lharris\PycharmProjects\road_condition_py3\venv\lib\site-packages\dill\_dill.py", line 893, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "C:\Python36\lib\pickle.py", line 821, in save_dict
self._batch_setitems(obj.items())
File "C:\Python36\lib\pickle.py", line 852, in _batch_setitems
save(v)
File "C:\Python36\lib\pickle.py", line 496, in save
rv = reduce(self.proto)
TypeError: can't pickle SwigPyObject objects
それを見ると、Pathos は Dill と Multiprocess を正しく使用しているように見えます。SwigPyObjects を並行して処理する方法はありますか?
数週間前に同様の質問がありましたが、回答が得られませんでした ( Python pathos error can't pickle SwigPyObject objects )。Python で copy_reg.pickle を使用していますか? )。
Windows 7でPython 3.6.5を使用してこのエラーが発生し、PyCharmで開発し、必要なパッケージがPyPiからダウンロードした.whlファイルからpipインストールされた仮想環境を使用しました（通常はpip installを使用できません私のネットワーク）。
誰かが私に与えることができるどんな助けも素晴らしいでしょう!