Forward/ballistic-builder-spring
2
0
Fork 0
mirror of https://gitea.gofwd.group/Forward_Group/ballistic-builder-spring.git synced 2026-01-20 16:51:03 -05:00
Code Issues Packages Projects Releases Wiki Activity

added minio and fixed repository code Slave fucked up

Browse Source
This commit is contained in:
Don Strawsburg
2025-12-18 11:19:51 -05:00
parent 1cfcf68f36
commit 12b3d2ae50
7 changed files with 275 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pom.xml
View File

@@ -172,6 +172,11 @@
<version>RELEASE</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>8.4.3</version>
</dependency>
</dependencies>
<build>

22
src/main/java/group/goforward/battlbuilder/configuration/MinioConfig.java Normal file
View File

@@ -0,0 +1,22 @@
package group.goforward.battlbuilder.configuration;
import io.minio.MinioClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class MinioConfig {
@Bean
public MinioClient minioClient(
@Value("${minio.endpoint}") String endpoint,
@Value("${minio.access-key}") String accessKey,
@Value("${minio.secret-key}") String secretKey
) {
return MinioClient.builder()
.endpoint(endpoint)
.credentials(accessKey, secretKey)
.build();
}
}

6
src/main/java/group/goforward/battlbuilder/model/Product.java
View File

@@ -74,6 +74,9 @@ public class Product {
@Column(name = "main_image_url")
private String mainImageUrl;
@Column(name = "battl_image_url")
private String battlImageUrl;
@Column(name = "created_at", nullable = false)
private Instant createdAt;
@@ -158,6 +161,9 @@ public class Product {
this.mainImageUrl = mainImageUrl;
}
public String getBattlImageUrl() {return battlImageUrl; }
public void setBattlImageUrl(String battlImageUrl) {this.battlImageUrl = battlImageUrl; }
public Instant getCreatedAt() { return createdAt; }
public void setCreatedAt(Instant createdAt) { this.createdAt = createdAt; }

21
src/main/java/group/goforward/battlbuilder/repos/ProductRepository.java
View File

@@ -1,12 +1,15 @@
package group.goforward.battlbuilder.repos;
import aj.org.objectweb.asm.commons.Remapper;
import group.goforward.battlbuilder.model.ImportStatus;
import group.goforward.battlbuilder.model.Brand;
import group.goforward.battlbuilder.model.Product;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.data.jpa.repository.JpaRepository;
import java.util.Collection;
import java.util.List;
@@ -230,4 +233,20 @@ public interface ProductRepository extends JpaRepository<Product, Integer> {
List<Product> findPendingMappingByMerchantId(@Param("merchantId") Integer merchantId);
Page<Product> findByDeletedAtIsNullAndMainImageUrlIsNotNullAndBattlImageUrlIsNull(Pageable pageable);
@Query("""
SELECT p
FROM Product p
WHERE p.deletedAt IS NULL
AND p.mainImageUrl IS NOT NULL
AND (
p.battlImageUrl IS NULL
OR TRIM(p.battlImageUrl) = ''
)
""")
Page<Product> findNeedingBattlImageUrlMigration(Pageable pageable);
}

183
src/main/java/group/goforward/battlbuilder/services/utils/ImageUrlToMinioMigrator.java Normal file
View File

@@ -0,0 +1,183 @@
package group.goforward.battlbuilder.services.utils;
import group.goforward.battlbuilder.model.Product;
import group.goforward.battlbuilder.repos.ProductRepository;
import io.minio.BucketExistsArgs;
import io.minio.MakeBucketArgs;
import io.minio.MinioClient;
import io.minio.PutObjectArgs;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.InputStream;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.Locale;
import java.util.Optional;
@Service
public class ImageUrlToMinioMigrator {
private final ProductRepository productRepository;
private final MinioClient minioClient;
private final String bucket;
private final String publicBaseUrl;
private final HttpClient httpClient = HttpClient.newBuilder()
.followRedirects(HttpClient.Redirect.NORMAL)
.connectTimeout(Duration.ofSeconds(15))
.build();
public ImageUrlToMinioMigrator(ProductRepository productRepository,
MinioClient minioClient,
@Value("${minio.bucket}") String bucket,
@Value("${minio.public-base-url}") String publicBaseUrl) {
this.productRepository = productRepository;
this.minioClient = minioClient;
this.bucket = bucket;
this.publicBaseUrl = trimTrailingSlash(publicBaseUrl);
}
/**
* Migrates Product.mainImageUrl URLs to MinIO and rewrites mainImageUrl to the new location.
*
* @param pageSize batch size for DB paging
* @param dryRun if true: download+upload is skipped and DB is not updated
* @param maxItems optional cap for safety (null = no cap)
* @return count of successfully migrated products
*/
@Transactional
public int migrateMainImages(int pageSize, boolean dryRun, Integer maxItems) {
ensureBucketExists();
int migrated = 0;
int page = 0;
while (true) {
if (maxItems != null && migrated >= maxItems) break;
Page<Product> batch = productRepository
.findNeedingBattlImageUrlMigration(PageRequest.of(page, pageSize));
if (batch.isEmpty()) break;
for (Product p : batch.getContent()) {
if (maxItems != null && migrated >= maxItems) break;
String sourceUrl = p.getMainImageUrl();
if (sourceUrl == null || sourceUrl.isBlank()) continue;
// Extra safety: skip if already set (covers any edge cases outside the query)
if (p.getBattlImageUrl() != null && !p.getBattlImageUrl().trim().isEmpty()) continue;
try {
if (!dryRun) {
String newUrl = uploadFromUrlToMinio(p, sourceUrl);
p.setBattlImageUrl(newUrl);
productRepository.save(p);
}
migrated++;
} catch (Exception ex) {
// fail-soft: continue migrating other products
}
}
if (!batch.hasNext()) break;
page++;
}
return migrated;
}
private String uploadFromUrlToMinio(Product p, String sourceUrl) throws Exception {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(sourceUrl))
.timeout(Duration.ofSeconds(60))
.GET()
.build();
HttpResponse<InputStream> response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
int status = response.statusCode();
if (status < 200 || status >= 300) {
throw new IllegalStateException("Failed to download image. HTTP " + status + " url=" + sourceUrl);
}
String contentType = response.headers()
.firstValue("content-type")
.map(v -> v.split(";", 2)[0].trim())
.orElse("application/octet-stream");
long contentLength = response.headers()
.firstValue("content-length")
.flatMap(ImageUrlToMinioMigrator::parseLongSafe)
.orElse(-1L);
String ext = extensionForContentType(contentType);
// Store under a stable key; adjust if you want per-merchant, hashed names, etc.
String objectName = "products/" + p.getId() + "/main" + ext;
try (InputStream in = response.body()) {
PutObjectArgs.Builder put = PutObjectArgs.builder()
.bucket(bucket)
.object(objectName)
.contentType(contentType);
if (contentLength >= 0) {
put.stream(in, contentLength, -1);
} else {
put.stream(in, -1, 10L * 1024 * 1024);
}
minioClient.putObject(put.build());
}
return publicBaseUrl + "/" + bucket + "/" + objectName;
}
private void ensureBucketExists() {
try {
boolean exists = minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucket).build());
if (!exists) {
minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucket).build());
}
} catch (Exception e) {
throw new IllegalStateException("Unable to ensure MinIO bucket exists: " + bucket, e);
}
}
private boolean looksAlreadyMigrated(String url) {
String prefix = publicBaseUrl + "/" + bucket + "/";
return url.startsWith(prefix);
}
private static Optional<Long> parseLongSafe(String v) {
try {
return Optional.of(Long.parseLong(v));
} catch (Exception e) {
return Optional.empty();
}
}
private static String extensionForContentType(String contentType) {
String ct = contentType.toLowerCase(Locale.ROOT);
if (ct.equals("image/jpeg") || ct.equals("image/jpg")) return ".jpg";
if (ct.equals("image/png")) return ".png";
if (ct.equals("image/webp")) return ".webp";
if (ct.equals("image/gif")) return ".gif";
if (ct.equals("image/svg+xml")) return ".svg";
return ".bin";
}
private static String trimTrailingSlash(String s) {
if (s == null) return "";
return s.endsWith("/") ? s.substring(0, s.length() - 1) : s;
}
}

28
src/main/java/group/goforward/battlbuilder/services/utils/MigrateProductImagesToMinioRunner.java Normal file
View File

@@ -0,0 +1,28 @@
package group.goforward.battlbuilder.services.utils;
import org.springframework.boot.CommandLineRunner;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Component;
@Component
@Profile("migrate-images-to-minio")
public class MigrateProductImagesToMinioRunner implements CommandLineRunner {
private final ImageUrlToMinioMigrator migrator;
public MigrateProductImagesToMinioRunner(ImageUrlToMinioMigrator migrator) {
this.migrator = migrator;
}
@Override
public void run(String... args) {
// Tune as needed. Start small; you can remove maxItems once you're confident.
int migrated = migrator.migrateMainImages(
200, // pageSize
false, // dryRun
1000 // maxItems safety cap
);
System.out.println("Migrated product images: " + migrated);
}
}

12
src/main/resources/application.properties
View File

@@ -35,4 +35,14 @@ spring.mail.properties.mail.smtp.starttls.required=true
#Database settings
spring.datasource.hikari.max-lifetime=600000
spring.datasource.hikari.max-lifetime=600000
minio.endpoint=https://minio.dev.gofwd.group
minio.access-key=<MINIO_ACCESS_KEY>
minio.secret-key=<MINIO_SECRET_KEY>
minio.bucket=battlbuilds
# Public base URL used to write back into products.main_image_url
minio.public-base-url=https://minio.dev.gofwd.group