![python 运行代码时使用整个 RAM](https://linux22.com/image/219402/python%20%E8%BF%90%E8%A1%8C%E4%BB%A3%E7%A0%81%E6%97%B6%E4%BD%BF%E7%94%A8%E6%95%B4%E4%B8%AA%20RAM.png)
我想问题不在于Python,而在于Linux 中的整体RAM 管理。所以这是代码:
!pip install numpy opencv-python pandas matplotlib tensorflow scikit-learn
import numpy as np
import cv2 as cv
import pandas as pd
import matplotlib.pyplot as plt
import os
import gc
gc.enable()
train_dir = 'fruits-360/Training'
classes = os.listdir(train_dir)
classes = classes[:30]
all_arrays=[]
img_size=100
for i in classes:
path=os.path.join(train_dir, i)
class_num=classes.index(i)
for img in os.listdir(path):
img_array=cv.imread(os.path.join(path, img))
mig_array=cv.cvtColor(img_array, cv.COLOR_BGR2RGB)
all_arrays.append([img_array, class_num])
test_dir = 'fruits-360/Test'
classes2 = os.listdir(test_dir)
classes2 = classes2[:30]
all_arrays2=[]
img_size=100
for i in classes2:
path=os.path.join(test_dir, i)
class_num2=classes.index(i)
for img in os.listdir(path):
img_array=cv.imread(os.path.join(path, img))
mig_array=cv.cvtColor(img_array, cv.COLOR_BGR2RGB)
all_arrays.append([img_array, class_num2])
import random
random.shuffle(all_arrays)
X_train=[]
Y_train=[]
for features, label in all_arrays:
X_train.append(features)
Y_train.append(label)
X_train=np.array(X_train)
random.shuffle(all_arrays2)
X_test=[]
Y_test=[]
for features, label in all_arrays:
X_test.append(features)
Y_test.append(label)
X_test=np.array(X_test)
X_train=X_train.reshape(-1, img_size, img_size, 3)
X_train=X_train/255
X_test=X_test.reshape(-1, img_size, img_size, 3)
X_test=X_test/255
from keras.utils import to_categorical
Y_train=to_categorical(Y_train, num_classes=30)
Y_test=to_categorical(Y_test, num_classes=30)
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
from sklearn.model_selection import train_test_split
from keras.models import Sequential
from keras.layers import Dense, Conv2D, Flatten, MaxPool2D, Dropout
from keras.callbacks import ReduceLROnPlateau
from keras.optimizers import Adam
x_train, x_val, y_train, y_val = train_test_split(X_train, Y_train, test_size=0.3, random_state=42)
model=Sequential()
model.add(Conv2D(filters=16, kernel_size=(5,5), padding='Same', activation='relu', input_shape=(100, 100, 3)))
model.add(MaxPool2D(pool_size=(2,2)))
model.add(Conv2D(filters=32, kernel_size=(5,5), padding='Same', activation='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
model.add(Conv2D(filters=64, kernel_size=(5,5), padding='Same', activation='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.6))
model.add(Dense(30, activation='softmax'))
model.compile(optimizer=Adam(learning_rate=0.001),
loss='categorical_crossentropy', metrics=['accuracy'])
epochs=10
batch_size = 32
history=model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs)
y_pred = model.predict(x_train)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_train, axis=1)
conf_mat = confusion_matrix(y_true, y_pred_classes)
disp = ConfusionMatrixDisplay(conf_mat, display_labels=classes)
fig, ax = plt.subplots(figsize=(15,15))
disp.plot(ax=ax)
plt.xticks(rotation=90)
plt.show()
model.summary()
问题不在于代码本身,以防万一。当我在 Windows 上的 jupyter 笔记本上运行此代码时,总体 RAM 使用量约为 9/16GB,并且代码运行良好,但是如果我在 Linux 上运行代码,它会消耗所有可用的 RAM 和交换分区,然后 jupyter 崩溃。如果我使用命令运行 jupyter 笔记本:
systemd-run --scope -p MemoryMax=8192M jupyter-notebook
Jupyter 在达到 8 GB 并使用整个交换空间后仍然崩溃。
有办法以某种方式修复它吗?
答案1
我注意到您在一开始就启用了垃圾收集,但gc
只会为不再在范围内的事物释放内存,并且按照您的布局方式,所有内容似乎都保留在范围内。可以帮助您找出问题的几种方法是:
- 将代码组合成更离散的函数。即,将 all_array 构建器逻辑和模型构建器逻辑分解为单独的函数,然后从
main
.前任:def parse_classes(dir_path, ref_classes): out_arrays = [] for i in ref_classes: ... return out_arrays train_arrays = parse_classes(train_dir, classes) test_arrays = parse_classes(test_dir, classes) def shuffle(arr): random.shuffle(arr) for features, labels, in arr: ... return X_train, Y_train # etc...
- 并使用
tracemalloc
帮助您确定每个区域使用了多少内存,以便您可以更好地了解正在发生的情况。这可以帮助您快速启动该过程。