Skip to content

I
Im
Commit 46cb23a3 by 王肇一
Options

should work, need test from SunBo

parent 8679ba65
Showing with 92 additions and 202 deletions
.gitignore
img/*
out/*
__pycache__/
.idea/*
data/imgs/*
data/masks/*
data/output/*
\ No newline at end of file
data/output/*
data/train_imgs/*
\ No newline at end of file
UnetBasedMethod.py
......@@ -2,32 +2,83 @@
# -*- coding:utf-8 -*-
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import torch
from PIL import Image
import cv2 as cv
from unet import UNet
from tqdm import tqdm
from multiprocessing import Pool
import math
import argparse
import logging
import os
import re
from unet import UNet
from utils.predict import predict_img
from resCalc import draw_bar,save_img,calcRes,get_subarea_info
from resCalc import save_img, get_subarea_info
def divide_list(list, process_num):
res = []
step = math.ceil(len(list) / process_num)
for i in range(0, len(list), step):
res.append(list[i:i + step])
return res
def step_1(net, args, device, list, position):
for fn in tqdm(list, position = position):
logging.info("\nPredicting image {} ...".format(fn[0] + '/' + fn[1]))
img = Image.open('data/imgs/' + fn[0] + '/' + fn[1])
mask = predict_img(net = net, full_img = img, scale_factor = args.scale, out_threshold = args.mask_threshold,
device = device)
result = (mask * 255).astype(np.uint8)
save_img({'ori': img, 'mask': result}, fn[0], fn[1])
try:
os.makedirs('data/masks/' + fn[0])
except:
logging.info("path already exist")
cv.imwrite('data/masks/' + fn[0] + '/' + fn[1], result)
def step_2(list, position):
for dir in list:
df = pd.DataFrame(columns = ('ug', 'iter', 'id', 'size', 'area_mean', 'back_mean'))
for name in tqdm(filter(lambda x: x!='.DS_Store', os.listdir('data/imgs/'+dir)),position = position):
match_group = re.match('.*\s([dD]2[oO]|[lL][bB]|.*ug).*\s(.+)\.tif', name)
img = cv.imread('data/imgs/' + dir + '/' + name, 0)
mask = cv.imread('data/masks/' + dir + '/' + name, 0)
tmp_df = pd.DataFrame(get_subarea_info(img, mask))
tmp_df['ug'] = str.lower(match_group.group(1))[:-2] if str.lower(match_group.group(1)).endswith(
'ug') else str.lower(match_group.group(1))
tmp_df['iter'] = str.lower(match_group.group(2))
df = df.append(tmp_df, ignore_index = True, sort = True)
df['pure'] = df['area_mean'] - df['back_mean']
df.sort_values('ug',inplace = True)
baseline = df[df['ug']=='d2o']['pure'].mean()*0.62
sns.set_style("darkgrid")
sns.catplot(x = 'ug', y = 'pure', kind = 'bar', data = df)
sns.lineplot(x = 'ug', y=baseline,data = df)
plt.savefig('data/output/'+dir+'.png')
def get_args():
parser = argparse.ArgumentParser(description = 'A simple toolkit designed by Ulden',
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--step', '-s', type = int, default = 1,help = "step 1: recognize; step 2: barcharts")
parser.add_argument('--step', '-s', type = int, default = 2, help = "step 1: recognize; step 2: barcharts")
parser.add_argument('-p', '--process', metavar = 'P', type = int, default = 8, help = 'Num of process',
dest = 'process')
# Unet para
parser.add_argument('--module', '-m', default = 'data/module/MODEL.pth', metavar = 'FILE',
parser.add_argument('--module', '-m', default = 'MODEL', metavar = 'FILE',
help = "Specify the file in which the model is stored")
parser.add_argument('--mask-threshold', '-t', type = float,
help = "Minimum probability value to consider a mask pixel white", default = 0.5)
parser.add_argument('--scale', '-S', type = float, help = "Scale factor for the input images", default = 0.5)
parser.add_argument('--scale', '-S', type = float, help = "Scale factor for the input images", default = 1.0)
return parser.parse_args()
......@@ -38,6 +89,7 @@ if __name__ == '__main__':
path = [(y, x) for y in filter(lambda x: x != '.DS_Store', os.listdir('data/imgs')) for x in filter(
lambda x: x.endswith('.tif') and not x.endswith('dc.tif') and not x.endswith('DC.tif') and not x.endswith(
'dc .tif'), os.listdir('data/imgs/' + y))]
seperate_path = divide_list(path, args.process)
if args.step == 1:
net = UNet(n_channels = 1, n_classes = 1)
......@@ -45,31 +97,22 @@ if __name__ == '__main__':
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
logging.info(f'Using device {device}')
net.to(device = device)
net.load_state_dict(torch.load(args.module, map_location = device))
net.load_state_dict(torch.load('data/module/' + args.module + '.pth', map_location = device))
logging.info("Model loaded !")
for i, fn in enumerate(tqdm(path)):
logging.info("\nPredicting image {} ...".format(fn[0] + '/' + fn[1]))
img = Image.open('data/imgs/' + fn[0] + '/' + fn[1])
mask = predict_img(net = net, full_img = img, scale_factor = args.scale, out_threshold = args.mask_threshold,
device = device)
result = (mask * 255).astype(np.uint8)
save_img({'ori': img, 'mask': result}, fn[0], fn[1])
try:
os.makedirs('data/masks/'+fn[0])
except:
logging.info("path already exist")
cv.imwrite('data/masks/' + fn[0] + '/' + fn[1],result)
#calcRes(cv.cvtColor(np.asarray(img), cv.COLOR_RGB2BGR), result, fn[0], fn[1][:-4])
elif args.step == 2:
df = pd.DataFrame(columns = ('ug','iter', 'id', 'size', 'area_mean', 'back_mean'))
for i,fn in enumerate(tqdm(path)):
img = cv.imread('data/imgs/' + fn[0] + '/' + fn[1],0)
mask = cv.imread('data/masks/'+ fn[0] + '/' + fn[1],0)
calcRes(img, mask, fn[0], fn[1][:-4])
pool = Pool(args.process)
for i, list in enumerate(seperate_path):
pool.apply_async(step_1, args = (net, args, device, list, i))
pool.close()
pool.join()
elif args.step == 2:
dir = [x for x in filter(lambda x: x != '.DS_Store', os.listdir('data/imgs/'))]
sep_dir = divide_list(dir,args.process)
step_2(dir,0)
#for exName in filter(lambda x: x != '.DS_Store', os.listdir('data/output')):
# draw_bar(exName, os.listdir('data/output/' + exName + '/csv'))
# pool = Pool(args.process)
# for i, list in enumerate(sep_dir):
# pool.apply_async(step_2, args = (list, i))
# pool.close()
# pool.join()
resCalc.py
......@@ -4,7 +4,6 @@ import pandas as pd
import numpy as np
import cv2 as cv
import matplotlib.pyplot as plt
import seaborn as sns
import logging
import os
import re
......@@ -28,100 +27,28 @@ def save_img(img_list, dir, name):
plt.close()
def calcRes(img, mask, dir = 'default_output', name = 'output'):
dic = get_subarea_info(img, mask)
df = pd.DataFrame(dic)
try:
os.makedirs('data/output/' + dir + '/csv')
except:
logging.info('Existing dir: out/' + dir + '/csv')
if len(df) != 0:
df.to_csv('data/output/' + dir + '/csv/' + name + '.csv', index = False)
def merge_df(dir):
df = pd.DataFrame(columns = ('ug', 'iter', 'id', 'size', 'area_mean', 'back_mean'))
for name in os.listdir(dir):
match_group = re.match('.*\s([dD]2[oO]|[lL][bB]|.*ug).*\s(.+)\.tif', name)
tmp = pd.read_csv('data/output/' + dir + '/csv/' + name)
tmp['ug'] = str.lower(match_group.group(1))[:-2] if str.lower(match_group.group(1)).endswith(
'ug') else str.lower(match_group.group(1))
tmp['iter'] = str.lower(match_group.group(2))
df = df.append(tmp, ignore_index = True, sort = True)
def draw_bar(exName, names):
df = pd.DataFrame(columns = ('class', 'perc', 'Label', 'Area', 'Mean', 'Std', 'BackMean', 'BackStd'))
for name in names:
tmp = pd.read_csv('data/output/' + exName + '/csv/' + name)
match_group = re.match('.*\s([dD]2[oO]|[lL][bB]|.*ug).*\s(.+)\.csv', name)
tmp['perc'] = str.lower(match_group.group(1))[:-2] if str.lower(match_group.group(1)).endswith(
'ug') else str.lower(match_group.group(1))
tmp['perc'].replace({'d2o': '0'}, inplace = True)
tmp['class'] = str.lower(match_group.group(2))
df = df.append(tmp, ignore_index = True, sort = True)
df = df[df['Area'] > 19]
df['Pure'] = df['Mean'] - df['BackMean']
sns.set_style("darkgrid")
# sns.catplot(x = 'perc',y = 'Mean',hue = 'class',kind='bar',data = df)
sns.pairplot(df, vars = ['Area', 'Mean', 'perc', 'class'])
plt.show()
# def get_subarea_infor(img, mask):
# area_num, labels, stats, centroids = cv.connectedComponentsWithStats(mask)
# info = []
#
# for i in filter(lambda x: x != 0, range(area_num)):
# group = np.where(labels == i)
# img_value = img[group]
# area_tmp = len(group[0])
# mean_tmp = np.mean(img_value)
# std_tmp = np.std(img_value)
#
# pos = [(group[0][i], group[1][i]) for i in range(len(group[0]))]
# res = np.zeros([200, 200], np.uint8)
# for x in range(200):
# for y in range(200):
# if (x, y) in pos:
# res[x, y] = mask[x, y]
# else:
# res[x, y] = 0
#
# kernel = np.ones((17, 17), np.uint8)
# mask_background = cv.erode(255 - res, kernel)
# minimask = cv.bitwise_xor(mask_background, 255 - res)
# realminimask = cv.bitwise_and(minimask, 255 - mask)
#
# img_background = img[np.where(realminimask != 0)]
# mean_value = np.mean(img_background)
# std_value = np.std(img_background)
#
# info.append({'Label': i, 'Area': area_tmp, 'Mean': mean_tmp, 'Std': std_tmp, 'BackMean': mean_value,
# 'BackStd': std_value})
# return info
def get_subarea_info(img, mask):
area_num, labels, stats, centroids = cv.connectedComponentsWithStats(mask, connectivity = 8)
info = []
for i in filter(lambda x: x != 0, range(area_num)):
group = np.where(labels == i)
area_value = img[group]
area_size = len(area_value)
area_mean = np.mean(area_value)
area_size = len(group[0])
if area_size > 10: # 过小的区域直接剔除
area_value = img[group]
area_mean = np.mean(area_value)
# Background Value
pos = [(group[0][k], group[1][k]) for k in range(len(group[0]))]
# Background Value
pos = [(group[0][i], group[1][i]) for i in range(len(group[0]))]
area_points = np.array([mask[x, y] if (x, y) in pos else 0 for x in range(200) for y in range(200)])
kernel = np.ones((15, 15), np.uint8)
bg_area_mask = cv.erode(area_points, kernel)
surround_bg_mask = cv.bitwise_xor(bg_area_mask, 255 - area_points)
real_bg_mask = cv.bitwise_and(surround_bg_mask, 255 - mask)
area_points = np.array([mask[x, y] if (x, y) in pos else 0 for x in range(200) for y in range(200)],dtype = np.uint8).reshape([200,200])
kernel = np.ones((15, 15), np.uint8)
bg_area_mask = cv.erode(area_points, kernel)
surround_bg_mask = cv.bitwise_xor(bg_area_mask, 255 - area_points)
real_bg_mask = cv.bitwise_and(surround_bg_mask, 255 - mask)
back_value = img[np.where(real_bg_mask != 0)]
back_mean = np.mean(back_value)
back_value = img[np.where(real_bg_mask != 0)]
back_mean = np.mean(back_value)
info.append({'id': i, 'size': area_size, 'area_mean': area_mean, 'back_mean': back_mean})
info.append({'id': i, 'size': area_size, 'area_mean': area_mean, 'back_mean': back_mean})
return info
server.py deleted 100644 → 0
#!/usr/bin/env python
# -*- coding:utf-8 -*-
from flask import Flask, render_template, url_for
app = Flask("Im")
@app.route('/')
def layout():
return render_template('layout.html')
@app.route('/compute/')
def filetree():
return 'filetree'
if __name__ == '__main__':
app.run()
templates/layout.html deleted 100644 → 0
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<link rel="stylesheet" href="https://unpkg.com/purecss@1.0.0/build/pure-min.css"
integrity="sha384-CCTZv2q9I9m3UOxRLaJneXrrqKwUNOzZ6NGEUMwHtShDJ+nCoiXJCAgi05KfkLGY" crossorigin="anonymous">
<script src="https://apps.bdimg.com/libs/jquery/2.1.4/jquery.min.js"></script>
<script src="../static/echarts.js"></script>
<title>main</title>
</head>
<body>
<div class="pure-g">
<div class="pure-u-1-3">
{% include 'tree.html' %}
</div>
<div class="pure-u-1-3">
</div>
<div class="pure-u-1-3">
</div>
</div>
<script type="text/javascript">
</script>
</body>
</html>
\ No newline at end of file
templates/tree.html deleted 100644 → 0
<ul id="browser" class="filetree treeview-famfamfam">
<li><span class="folder">Folder 1</span>
<ul>
<li><span class="folder">Item 1.1</span>
<ul>
<li><span class="file">Item 1.1.1</span></li>
</ul>
</li>
<li><span class="folder">Folder 2</span>
<ul>
<li><span class="folder">Subfolder 2.1</span>
<ul id="folder21">
<li><span class="file">File 2.1.1</span></li>
<li><span class="file">File 2.1.2</span></li>
</ul>
</li>
<li><span class="folder">Subfolder 2.2</span>
<ul>
<li><span class="file">File 2.2.1</span></li>
<li><span class="file">File 2.2.2</span></li>
</ul>
</li>
</ul>
</li>
<li class="closed"><span class="folder">Folder 3 (closed at start)</span>
<ul>
<li><span class="file">File 3.1</span></li>
</ul>
</li>
<li><span class="file">File 4</span></li>
</ul>
</li>
</ul>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment