Skip to content

X
xzxt-api-report
Commit 817b7eac by dupengfei
Options

兵团 在逃人员zip照片解析入库 实现

parent d1df71fd
Showing with 631 additions and 45 deletions
pom.xml
......@@ -26,47 +26,27 @@
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
<version>2.1.7.RELEASE</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>cn.com.winmage</groupId>
......@@ -99,6 +79,13 @@
<version>${guava.version}</version>
</dependency>
<!--zip-->
<dependency>
<groupId>net.lingala.zip4j</groupId>
<artifactId>zip4j</artifactId>
<version>2.10.0</version>
</dependency>
<!-- 数据库驱动包 -->
<dependency>
<groupId>org.oracle</groupId>
......@@ -125,12 +112,7 @@
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>1.3.2</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- ftp -->
<dependency>
......
src/main/java/com/founder/config/SchedulingConfig.java 0 → 100644
package com.founder.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.SchedulingConfigurer;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
import org.springframework.scheduling.config.ScheduledTaskRegistrar;
/*
* @Description TODO
* @Author du_pengfei
* @Date 2022/4/11
* @Version 1.0
*/
@Configuration
@EnableScheduling
public class SchedulingConfig implements SchedulingConfigurer {
@Bean
public ThreadPoolTaskScheduler scheduler(){
ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
scheduler.setPoolSize(10);
return scheduler;
}
@Override
public void configureTasks(ScheduledTaskRegistrar registrar) {
registrar.setScheduler(scheduler());
}
}
src/main/java/com/founder/controller/FaceController.java
......@@ -7,17 +7,19 @@ import com.founder.model.Face;
import com.founder.model.Jccj;
import com.founder.service.AutoSeqService;
import com.founder.service.FaceService;
import com.founder.service.PicToDatabaseAndCopy;
import com.founder.service.impl.JccjServiceImpl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import javax.servlet.http.HttpServletResponse;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.net.URLEncoder;
import java.util.Base64;
import java.util.List;
import java.util.Map;
......@@ -25,6 +27,10 @@ import java.util.Map;
public class FaceController {
@Autowired
private FaceService faceService;
@Autowired
private PicToDatabaseAndCopy service;
@ResponseBody
@RequestMapping("/query1")
public String query() {
......@@ -75,4 +81,18 @@ public class FaceController {
return "已执行";
}
@ResponseBody
@RequestMapping("/query2")
public String query2() throws Exception {
System.out.println("=======");
service.saveZtryPic();
System.out.println("=======");
return "1";
}
}
src/main/java/com/founder/dao/PicToDataBaseAndCopyDao.java 0 → 100644
package com.founder.dao;
import com.founder.model.TbYwZtry;
import org.apache.ibatis.annotations.Mapper;
import org.springframework.stereotype.Component;
/*
* @Description TODO
* @Author du_pengfei
* @Date 2022/4/12
* @Version 1.0
*/
@Mapper
@Component
public interface PicToDataBaseAndCopyDao {
int selectProcessedCount(String fileName);
int updateZtryPic(TbYwZtry ztry);
int insertProcessedZip(String name);
}
src/main/java/com/founder/model/TbYwZtry.java 0 → 100644
package com.founder.model;
import lombok.Data;
/*
* @Description TODO
* @Author du_pengfei
* @Date 2022/4/12
* @Version 1.0
*/
@Data
public class TbYwZtry {
private String rybh;
private byte[] picContent;
private String zp;
public void clear(){
rybh = "";
picContent = null;
zp = "";
}
}
src/main/java/com/founder/service/PicToDatabaseAndCopy.java 0 → 100644
package com.founder.service;
import java.io.IOException;
/*
* @Description TODO
* @Author du_pengfei
* @Date 2022/4/11
* @Version 1.0
*/
public interface PicToDatabaseAndCopy {
//保存在逃人员照片到数据库
void saveZtryPic() throws IOException, Exception;
//备份在逃人员照片
void copyZtryPic() throws IOException;
}
src/main/java/com/founder/service/impl/PicToDataBaseAndCopyImp.java 0 → 100644
package com.founder.service.impl;
import com.founder.dao.PicToDataBaseAndCopyDao;
import com.founder.model.TbYwZtry;
import com.founder.service.PicToDatabaseAndCopy;
import net.lingala.zip4j.ZipFile;
import net.lingala.zip4j.exception.ZipException;
import net.lingala.zip4j.io.inputstream.ZipInputStream;
import net.lingala.zip4j.model.FileHeader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
/*
* @Description TODO
* @Author du_pengfei
* @Date 2022/4/11
* @Version 1.0
*/
@Service
public class PicToDataBaseAndCopyImp implements PicToDatabaseAndCopy {
Logger logger = LoggerFactory.getLogger(PicToDataBaseAndCopyImp.class);
@Autowired
PicToDataBaseAndCopyDao dao;
@Value("${picPath}")
private String localPath;
@Value("${backupPath}")
private String copyPath;
@Scheduled(cron = "${cron}")
@Override
public void saveZtryPic() throws Exception {
Files.newDirectoryStream(Paths.get(localPath), path -> path.toString().endsWith("_ZP.zip")).forEach(path -> {
//压缩包名称
String fileName = path.getFileName().toString();
try {
//读取zip文件并入库
readZipFile(path.toString(), fileName);
} catch (Exception e) {
logger.error(e.getMessage());
}
});
copyFile();
}
//@Scheduled()
@Override
public void copyZtryPic() throws IOException {
}
public void copyFile() throws IOException{
//如果没有这个目录就创建
if (Files.notExists(Paths.get(copyPath))){
Files.createDirectories(Paths.get(copyPath));
}
Files.list(Paths.get(localPath)).forEach(path -> {
String fileName = path.getFileName().toString();
try {
Files.move(path, Paths.get(copyPath+fileName), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
} catch (IOException e) {
logger.error("文件拷贝失败: {}",e.getMessage());
//e.printStackTrace();
}
});
}
/**
* @param directoryPathStr 文件夹路径
* @return
*/
public boolean deleteDirectory(String directoryPathStr) {
Path reportPath = Paths.get(directoryPathStr);
//先遍历删除文件夹中文件，再删除文件夹
if (Files.exists(reportPath)) {
try {
List<File> directoryList = new ArrayList<>();
Files.walk(reportPath).map(Path::toFile).forEach(file -> {
if (file.isFile()) {
//文件直接删除
file.delete();
} else {
directoryList.add(file);
}
});
//设备最大循环次数，防止死循环
int time = 0, maxTime = directoryList.size();
while (directoryList.size() > 0 && time < maxTime) {
//遍历删除
directoryList.removeIf(file -> file.list().length == 0 && file.delete());
time++;
}
} catch (Exception e) {
return false;
}
}
return true;
}
public void readZipFile(String file, String fileName) throws Exception {
ByteArrayOutputStream stream = new ByteArrayOutputStream();
ZipFile zFile = new ZipFile(file);
// 此处最好立即设置字符集
zFile.setCharset(Charset.forName("gbk"));
if (!zFile.isValidZipFile()) {
logger.error("压缩文件不合法,可能被损坏,文件名 :{}", file);
Files.move(Paths.get(file), Paths.get(copyPath+ fileName+ ".notValid"),StandardCopyOption.REPLACE_EXISTING);
return ;
}
// 获取ZIP中所有文件的FileHeader，以便后面对zip中文件进行遍历
List<FileHeader> list = zFile.getFileHeaders();
// 此时list的size包括：文件夹、子文件夹、文件的个数
logger.info("{}, 文件个数为: {}", file, list.size());
//System.out.println(list.size());
// 遍历其中的文件
for (FileHeader fileHeader : list) {
ZipInputStream inputStream = zFile.getInputStream(fileHeader);
byte[] bytes = new byte[1024];
int len = 0;
// 将缓存区的数据读成byte数组
while ((len=inputStream.read(bytes)) != -1) {
stream.write(bytes, 0, len );
}
byte[] bytesArray = stream.toByteArray();
stream.reset();
insertDatabase(fileHeader, bytesArray);
inputStream.close();
}
zFile.close();
stream.close();
}
public void insertDatabase(FileHeader fileHeader, byte[] bytes) {
TbYwZtry ztry = new TbYwZtry();
String fileName = fileHeader.getFileName();
ztry.setPicContent(bytes);
String replace = fileName.replace(".jpg", "");
String[] nameList = replace.split("_");
for (String name : nameList){
if (name.length() == 23){
ztry.setRybh(name);
} else if (name.length() == 1){
ztry.setZp(name);
}
}
//更新人员照片
dao.updateZtryPic(ztry);
}
}
src/main/java/com/founder/util/FtpUtil.java
......@@ -7,6 +7,14 @@ import org.apache.commons.net.ftp.FTPReply;
import java.io.*;
import java.net.SocketException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;
/**
* ftp上传下载工具类
......@@ -197,6 +205,72 @@ public class FtpUtil {
/**
* Description: 获取zip文件中的文件
* @param path 文件路径
*
* @return
*/
public static List<String> readZipFile(String path){
List<String> list = new ArrayList<>();
List<List<String>> ddlList=null;
try {
ZipFile zipFile = new ZipFile(path);
InputStream in = new BufferedInputStream(new FileInputStream(path));
ZipInputStream zin = new ZipInputStream(in);
ZipEntry ze;
while ((ze = zin.getNextEntry()) != null) {
ddlList=new ArrayList<>();
if (ze.isDirectory()) {
} else {
System.err.println("file - " + ze.getName() + " : "+ ze.getSize() + " bytes");
long size = ze.getSize();
if (size > 0) {
BufferedReader br = new BufferedReader(new InputStreamReader(zipFile.getInputStream(ze), Charset.forName("gbk")));
String line;
while ((line = br.readLine()) != null) {
String[] index = line.split(",");
List<String> indexList = Arrays.asList(index);
ddlList.add(indexList);
}
br.close();
}
}
//处理ddlList,此时ddlList为每个文件的内容,while每循环一次则读取一个文件
}
zin.closeEntry();
} catch (IOException e) {
e.printStackTrace();
}
//此处返回无用,懒得修改了
return list;
}
/**
* Description: 获取zip包中的文件列表
* @param path FTP服务器hostname
*
* @return
*/
public static List<String> readZipFileName(String path){
List<String> list = new ArrayList<>();
try {
ZipFile zipFile = new ZipFile(path);
Enumeration<? extends ZipEntry> entries = zipFile.entries();
while (entries.hasMoreElements()) {
list.add(entries.nextElement().getName());
}
} catch (IOException e) {
e.printStackTrace();
}
return list;
}
/**
* 上传文件到ft，支持断点续传，文件名称和ftp路径应使用utf-8编码，避免乱码
* @param host
* @param port
......
src/main/java/com/founder/util/ZipUtil.java 0 → 100644
package com.founder.util;
import lombok.extern.slf4j.Slf4j;
import net.lingala.zip4j.exception.ZipException;
import net.lingala.zip4j.model.FileHeader;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
/*
* @Description TODO
* @Author du_pengfei
* @Date 2022/4/15
* @Version 1.0
*/
@Slf4j
public class ZipUtil {
public static void unZip(File zipFile, String outPath) {
}
}
src/main/resources/application-dev.yml
......@@ -2,15 +2,15 @@ spring:
datasource:
xzxt:
driver-class-name: oracle.jdbc.OracleDriver
username: YTHCJ
password: YTHCJ
jdbc-url: jdbc:oracle:thin:@10.142.16.172:1621:STXZ
username: ZTRY
password: ZTRY
jdbc-url: jdbc:oracle:thin:@107.0.64.6:6621:XZSJK
type: com.alibaba.druid.pool.DruidDataSource
manager:
driver-class-name: oracle.jdbc.OracleDriver
username: YTHCJ
password: YTHCJ
jdbc-url: jdbc:oracle:thin:@10.142.16.172:1621:STXZ
username: ZTRY
password: ZTRY
jdbc-url: jdbc:oracle:thin:@107.0.64.6:6621:XZSJK
type: com.alibaba.druid.pool.DruidDataSource
#配置jpa 使其展示sql语句
......@@ -21,11 +21,26 @@ spring:
port: 43079
password: Xzxt#4302
#----------------------兵团 在逃人员照片入库 begin ----------------#
#压缩包所在文件目录 记得路径后面加上反斜杠 \
picPath: C:\ZTZP\ztrychuli\
#文件备份目录 记得路径后面加上反斜杠 \
backupPath: C:\ZTZP\ztbf\
#文件解压后的目录 记得路径后面加上反斜杠 \
unZipPath: C:\ZTZP\temp\
#定时任务 每天中午 12 点
cron: 0 0 12 * * ?
#----------------------兵团 在逃人员照片入库 end ----------------#
fileDir: D:/fileDir/
outDir: D:/outDir/
#isCs: true
isCs: false
poorSize: 100
ftpserverip: 10.143.242.44
nginxport: 9053
ftpport: 4546+
ftpport: 4546
ftpuser: ftpuser
ftppass: fou3rfnder4SD1
ftpbathPath: C:/ftp/xzxt
......
src/main/resources/application-prod.yml
......@@ -3,8 +3,8 @@ spring:
xzxt:
driver-class-name: oracle.jdbc.OracleDriver
username: XZXT
password: XZXT
jdbc-url: jdbc:oracle:thin:@47.92.129.99:1600:orcl
password: XzPwd#11022
jdbc-url: jdbc:oracle:thin:@47.92.129.99:1560:orcl
type: com.alibaba.druid.pool.DruidDataSource
#配置jpa 使其展示sql语句
jpa:
......@@ -13,10 +13,28 @@ spring:
host: 47.92.223.200
port: 7779
password: p!ssw0rd1
resources: static-locations=classpath:/webapp/static/
#----------------------兵团 在逃人员照片入库 begin ----------------#
#压缩包所在文件目录 记得路径后面加上 /
picPath: /Users/mac/xzxt-api-report/target/classes/WEB-INF/static/file/
#文件备份目录 记得路径后面加上 /
backupPath: /Users/mac/xzxt-api-report/target/classes/WEB-INF/static/copy/
#文件解压后的目录 记得路径后面加上 /
unZipPath: /Users/mac/xzxt-api-report/target/classes/WEB-INF/static/temp/
#定时任务 每天中午 12 点
cron: 0 0 12 * * ?
#----------------------兵团 在逃人员照片入库 end ----------------#
fileDir: D:/fileDir/
outDir: D:/outDir/
#isCs: true
isCs: false
poorSize: 100
ftpserverip: 47.92.129.99
nginxport: 9053
ftpport: 4546
ftpuser: ftpuser
ftppass: p@ssw0rd[123]
......
src/main/resources/application.yml
spring:
profiles:
active: dev
active: prod
jackson:
date-format: yyyy-MM-dd HH:mm:ss
time-zone: GMT+8
......
src/main/resources/mybatis/mapper/PicToDatabaseAndCopyDao.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.founder.dao.PicToDataBaseAndCopyDao">
<update id="updateZtryPic" parameterType="com.founder.model.TbYwZtry">
update tb_yw_ztry
<set>
<if test="zp == 1"> zp1 = #{picContent} ,</if>
<if test="zp == 2"> zp2 = #{picContent} ,</if>
<if test="zp == 3"> zp3 = #{picContent} ,</if>
<if test="zp == 4"> zp4 = #{picContent} ,</if>
<if test="zp == 5"> zp5 = #{picContent} ,</if>
<if test="zp == 6"> zp6 = #{picContent} ,</if>
gxsj = sysdate
</set>
where 1=1 and ztrybh = #{rybh}
</update>
</mapper>
\ No newline at end of file
src/test/java/com/founder/ApplicationTests.java
package com.founder;
import com.founder.model.TbYwZtry;
import com.founder.service.impl.PicToDataBaseAndCopyImp;
import org.checkerframework.checker.units.qual.A;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
@RunWith(SpringRunner.class)
@SpringBootTest
import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.locks.ReentrantLock;
//@RunWith(SpringRunner.class)
//@SpringBootTest
public class ApplicationTests {
//@Autowired
//PicToDataBaseAndCopyImp imp;
public static volatile int num = 0;
public static int total = 0;
@Test
public void contextLoads() {
//imp.saveZtryPic();
}
@Test
public void test() {
String localPath = "/Users/mac/xzxt-api-report/src/main/webapp/WEB-INF/static";
String copyPath = "/Users/mac/xzxt-api-report/src/main/webapp/WEB-INF/static/copy/";
String unZipPath = "/Users/mac/xzxt-api-report/src/main/webapp/WEB-INF/static/temp/";
}
@Test
public void test2() throws Exception {
String localPath = "/Users/mac/xzxt-api-report/src/main/webapp/WEB-INF/static";
Files.newDirectoryStream(Paths.get(localPath), path -> path.toString().endsWith("_ZP.zip")).forEach(System.out::println);
}
@Test
public void test3() throws IOException, URISyntaxException {
String localPath = "/Users/mac/xzxt-api-report/src/main/webapp/WEB-INF/static";
String copyPath = "/Users/mac/xzxt-api-report/src/main/webapp/WEB-INF/static/copy/";
String unZipPath = "/Users/mac/xzxt-api-report/src/main/webapp/WEB-INF/static/temp/";
if (Files.notExists(Paths.get(copyPath))){
Files.createDirectories(Paths.get(copyPath));
}
TbYwZtry tbYwZtry = new TbYwZtry();
URL resource = tbYwZtry.getClass().getResource("src/main/resources");
System.out.println(resource);
//Files.delete(Paths.get(copyPath));
Files.copy(Paths.get("src/main/webapp/WEB-INF/static/20220410.zip"), Paths.get(copyPath+"20220410.zip"), StandardCopyOption.REPLACE_EXISTING);
}
@Test
public void thread(){
//Thread thread = new Thread();
//System.out.println(Runtime.getRuntime().availableProcessors());
Object o = new Object();
for (int i= 0 ; i < 10; i++){
Thread thread = new Thread(()->{
for (int j=0; j< 1000; j++){
synchronized (o){
num++; //分为三 读 加 写 步操作 多线程中不能保证其原子性 需要加锁
}
}
});
thread.start();
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println(num);
}
@Test
public void countdown() throws InterruptedException {
CountDownLatch countDownLatch = new CountDownLatch(1);
ReentrantLock reentrantLock = new ReentrantLock();
for (int i = 0; i < 10 ; i++){
new Thread(()->{
try {
countDownLatch.await();
for (int j= 0; j< 1000; j++){
total++;
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}).start();
}
Thread.sleep(2000);
countDownLatch.countDown();
Thread.sleep(2000);
System.out.println(total);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment