# Create output directory if it doesn't exist os.makedirs(output_dir, exist_ok=True)
# Get list of all .tif files in the input directory tif_files = [f for f in os.listdir(input_dir) if f.endswith('.tif')]
# Function to process each raster file def process_raster(input_path, output_path): with rasterio.open(input_path) as src: # Read metadata meta = src.meta.copy()
# Initialize the processed data array processed_data = np.zeros((src.height, src.width), dtype=np.uint8)
# Define block size (you can adjust this based on available memory) block_size = 1024 # 建议改256，减小块减小内存使用
# Calculate total number of blocks num_blocks_x = (src.width + block_size - 1) // block_size # Horizontal blocks num_blocks_y = (src.height + block_size - 1) // block_size # Vertical blocks
# Process the raster in blocks to reduce memory usage for i in tqdm(range(0, src.height, block_size), desc=f'Processing {os.path.basename(input_path)}', total=num_blocks_y): for j in range(0, src.width, block_size): window = rasterio.windows.Window(j, i, block_size, block_size) data = src.read(1, window=window)
# Create a mask for valid data (values between 11 and 99) mask = (data >= 11) & (data <= 99)
# Process data (compute tens digit) processed_data[i:i+block_size, j:j+block_size] = np.where(mask, data // 10, 0)
# Update metadata for output meta.update(dtype=rasterio.uint8, compress='LZW', tiled=True, bigtiff='YES')
# Write the processed data to the output file with high compression with rasterio.open(output_path, 'w', **meta) as dst: dst.write(processed_data, 1)
# Use ProcessPoolExecutor to process each .tif file in parallel def parallel_processing(): with ProcessPoolExecutor() as executor: # Map the process_raster function to each file in the list, and track progress with tqdm futures = [ executor.submit(process_raster, os.path.join(input_dir, tif_file), os.path.join(output_dir, tif_file)) for tif_file in tif_files ]
# Use tqdm to track the overall progress of the futures for future in tqdm(futures, desc="Overall progress", total=len(futures)): future.result() # This will raise exceptions if there were any errors during processing
# Run the parallel processing if __name__ == "__main__": parallel_processing() print("All files processed successfully.")
# Create output directory if it doesn't exist os.makedirs(output_dir, exist_ok=True)
# Get list of all .tif files in the input directory获取所有的tif文件 tif_files = [f for f in os.listdir(input_dir) if f.endswith('.tif')]
tif_files = tif_files[6:10]
# Function to process each raster file using GPU (with memory-efficient approach) def process_raster(input_path, output_path): with rasterio.open(input_path) as src: # Read metadata meta = src.meta.copy()
# Initialize the processed data array (on CPU) processed_data = np.zeros((src.height, src.width), dtype=np.uint8)
# Define block size (you can adjust this based on available memory) block_size = 1024 # You can modify this to a smaller block size if needed
# Calculate total number of blocks num_blocks_x = (src.width + block_size - 1) // block_size # Horizontal blocks
