Max Petrov
2 years ago
committed by
GitHub
GitHub
867 changed files with 6747 additions and 29030 deletions
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -0,0 +1,98 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.entitiy.dashboard; |
|||
|
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.ResourceType; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.queue.ServiceType; |
|||
import org.thingsboard.server.dao.resource.ResourceService; |
|||
import org.thingsboard.server.dao.widget.WidgetsBundleService; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
import org.thingsboard.server.queue.util.AfterStartUp; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.sync.GitSyncService; |
|||
import org.thingsboard.server.service.sync.vc.GitRepository.FileType; |
|||
import org.thingsboard.server.service.sync.vc.GitRepository.RepoFile; |
|||
|
|||
import java.util.List; |
|||
import java.util.concurrent.TimeUnit; |
|||
import java.util.stream.Stream; |
|||
|
|||
@Service |
|||
@TbCoreComponent |
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
@ConditionalOnProperty(value = "transport.gateway.dashboard.sync.enabled", havingValue = "true") |
|||
public class DashboardSyncService { |
|||
|
|||
private final GitSyncService gitSyncService; |
|||
private final ResourceService resourceService; |
|||
private final WidgetsBundleService widgetsBundleService; |
|||
private final PartitionService partitionService; |
|||
|
|||
@Value("${transport.gateway.dashboard.sync.repository_url:}") |
|||
private String repoUrl; |
|||
@Value("${transport.gateway.dashboard.sync.branch:main}") |
|||
private String branch; |
|||
@Value("${transport.gateway.dashboard.sync.fetch_frequency:24}") |
|||
private int fetchFrequencyHours; |
|||
|
|||
private static final String REPO_KEY = "gateways-dashboard"; |
|||
private static final String GATEWAYS_DASHBOARD_KEY = "gateways_dashboard.json"; |
|||
|
|||
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) |
|||
public void init() throws Exception { |
|||
gitSyncService.registerSync(REPO_KEY, repoUrl, branch, TimeUnit.HOURS.toMillis(fetchFrequencyHours), this::update); |
|||
} |
|||
|
|||
private void update() { |
|||
if (!partitionService.isMyPartition(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID)) { |
|||
return; |
|||
} |
|||
|
|||
List<RepoFile> resources = listFiles("resources"); |
|||
for (RepoFile resourceFile : resources) { |
|||
String data = getFileContent(resourceFile.path()); |
|||
resourceService.createOrUpdateSystemResource(ResourceType.JS_MODULE, resourceFile.name(), data); |
|||
} |
|||
|
|||
Stream<String> widgetsBundles = listFiles("widget_bundles").stream() |
|||
.map(widgetsBundleFile -> getFileContent(widgetsBundleFile.path())); |
|||
Stream<String> widgetTypes = listFiles("widget_types").stream() |
|||
.map(widgetTypeFile -> getFileContent(widgetTypeFile.path())); |
|||
widgetsBundleService.updateSystemWidgets(widgetsBundles, widgetTypes); |
|||
|
|||
RepoFile dashboardFile = listFiles("dashboards").get(0); |
|||
String dashboardJson = getFileContent(dashboardFile.path()); |
|||
resourceService.createOrUpdateSystemResource(ResourceType.DASHBOARD, GATEWAYS_DASHBOARD_KEY, dashboardJson); |
|||
|
|||
log.info("Gateways dashboard sync completed"); |
|||
} |
|||
|
|||
private List<RepoFile> listFiles(String path) { |
|||
return gitSyncService.listFiles(REPO_KEY, path, 1, FileType.FILE); |
|||
} |
|||
|
|||
private String getFileContent(String path) { |
|||
return gitSyncService.getFileContent(REPO_KEY, path); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,172 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync; |
|||
|
|||
import jakarta.annotation.PreDestroy; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.commons.lang3.StringUtils; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.common.util.ThingsBoardThreadFactory; |
|||
import org.thingsboard.server.common.data.sync.vc.RepositorySettings; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.sync.vc.GitRepository; |
|||
import org.thingsboard.server.service.sync.vc.GitRepository.FileType; |
|||
import org.thingsboard.server.service.sync.vc.GitRepository.RepoFile; |
|||
|
|||
import java.net.URI; |
|||
import java.nio.file.Path; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.Executors; |
|||
import java.util.concurrent.ScheduledExecutorService; |
|||
import java.util.concurrent.TimeUnit; |
|||
|
|||
@TbCoreComponent |
|||
@Service |
|||
@Slf4j |
|||
public class DefaultGitSyncService implements GitSyncService { |
|||
|
|||
@Value("${vc.git.repositories-folder:${java.io.tmpdir}/repositories}") |
|||
private String repositoriesFolder; |
|||
|
|||
private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("git-sync")); |
|||
private final Map<String, GitRepository> repositories = new ConcurrentHashMap<>(); |
|||
private final Map<String, Runnable> updateListeners = new ConcurrentHashMap<>(); |
|||
|
|||
@Override |
|||
public void registerSync(String key, String repoUri, String branch, long fetchFrequencyMs, Runnable onUpdate) { |
|||
RepositorySettings settings = new RepositorySettings(); |
|||
settings.setRepositoryUri(repoUri); |
|||
settings.setDefaultBranch(branch); |
|||
if (onUpdate != null) { |
|||
updateListeners.put(key, onUpdate); |
|||
} |
|||
|
|||
executor.execute(() -> { |
|||
initRepository(key, settings); |
|||
}); |
|||
|
|||
executor.scheduleWithFixedDelay(() -> { |
|||
GitRepository repository = repositories.get(key); |
|||
if (repository == null || !GitRepository.exists(repository.getDirectory())) { |
|||
initRepository(key, settings); |
|||
return; |
|||
} |
|||
|
|||
try { |
|||
log.debug("[{}] Fetching repository", key); |
|||
boolean updated = repository.fetch(); |
|||
if (updated) { |
|||
onUpdate(key); |
|||
} else { |
|||
log.debug("[{}] No changes in the repository", key); |
|||
} |
|||
} catch (Throwable e) { |
|||
log.error("[{}] Failed to fetch repository", key, e); |
|||
} |
|||
}, fetchFrequencyMs, fetchFrequencyMs, TimeUnit.MILLISECONDS); |
|||
} |
|||
|
|||
@Override |
|||
public List<RepoFile> listFiles(String key, String path, int depth, FileType type) { |
|||
GitRepository repository = getRepository(key); |
|||
return repository.listFilesAtCommit(getBranchRef(repository), path, depth).stream() |
|||
.filter(file -> type == null || file.type() == type) |
|||
.toList(); |
|||
} |
|||
|
|||
|
|||
@Override |
|||
public String getFileContent(String key, String path) { |
|||
GitRepository repository = getRepository(key); |
|||
try { |
|||
return repository.getFileContentAtCommit(path, getBranchRef(repository)); |
|||
} catch (Exception e) { |
|||
log.warn("[{}] Failed to get file content for path {}: {}", key, path, e.getMessage()); |
|||
return "{}"; |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public String getGithubRawContentUrl(String key, String path) { |
|||
if (path == null) { |
|||
return ""; |
|||
} |
|||
RepositorySettings settings = getRepository(key).getSettings(); |
|||
return StringUtils.removeEnd(settings.getRepositoryUri(), ".git") + "/blob/" + settings.getDefaultBranch() + "/" + path + "?raw=true"; |
|||
} |
|||
|
|||
private GitRepository getRepository(String key) { |
|||
GitRepository repository = repositories.get(key); |
|||
if (repository != null) { |
|||
if (!GitRepository.exists(repository.getDirectory())) { |
|||
// reinitializing the repository because folder was deleted
|
|||
initRepository(key, repository.getSettings()); |
|||
} |
|||
} |
|||
|
|||
repository = repositories.get(key); |
|||
if (repository == null) { |
|||
throw new IllegalStateException(key + " repository is not initialized"); |
|||
} |
|||
return repository; |
|||
} |
|||
|
|||
private void initRepository(String key, RepositorySettings settings) { |
|||
try { |
|||
repositories.remove(key); |
|||
Path directory = getRepoDirectory(settings); |
|||
|
|||
GitRepository repository = GitRepository.openOrClone(directory, settings, true); |
|||
repositories.put(key, repository); |
|||
log.info("[{}] Initialized repository", key); |
|||
|
|||
onUpdate(key); |
|||
} catch (Throwable e) { |
|||
log.error("[{}] Failed to initialize repository with settings {}", key, settings, e); |
|||
} |
|||
} |
|||
|
|||
private void onUpdate(String key) { |
|||
Runnable listener = updateListeners.get(key); |
|||
if (listener != null) { |
|||
log.debug("[{}] Handling repository update", key); |
|||
try { |
|||
listener.run(); |
|||
} catch (Throwable e) { |
|||
log.error("[{}] Failed to handle repository update", key, e); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private Path getRepoDirectory(RepositorySettings settings) { |
|||
// using uri to define folder name in case repo url is changed
|
|||
String name = URI.create(settings.getRepositoryUri()).getPath().replaceAll("[^a-zA-Z]", ""); |
|||
return Path.of(repositoriesFolder, name); |
|||
} |
|||
|
|||
private String getBranchRef(GitRepository repository) { |
|||
return "refs/remotes/origin/" + repository.getSettings().getDefaultBranch(); |
|||
} |
|||
|
|||
@PreDestroy |
|||
private void preDestroy() { |
|||
executor.shutdownNow(); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,65 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.update; |
|||
|
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.rule.engine.api.NotificationCenter; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.notification.info.GeneralNotificationInfo; |
|||
import org.thingsboard.server.common.data.notification.targets.platform.SystemAdministratorsFilter; |
|||
import org.thingsboard.server.dao.notification.DefaultNotifications; |
|||
import org.thingsboard.server.queue.util.AfterStartUp; |
|||
|
|||
import java.util.Map; |
|||
|
|||
@Service |
|||
@Slf4j |
|||
@RequiredArgsConstructor |
|||
public class DeprecationService { |
|||
|
|||
private final NotificationCenter notificationCenter; |
|||
|
|||
@Value("${queue.type}") |
|||
private String queueType; |
|||
|
|||
@AfterStartUp(order = Integer.MAX_VALUE) |
|||
public void checkDeprecation() { |
|||
checkQueueTypeDeprecation(); |
|||
} |
|||
|
|||
private void checkQueueTypeDeprecation() { |
|||
String queueTypeName; |
|||
switch (queueType) { |
|||
case "aws-sqs" -> queueTypeName = "AWS SQS"; |
|||
case "pubsub" -> queueTypeName = "PubSub"; |
|||
case "service-bus" -> queueTypeName = "Azure Service Bus"; |
|||
case "rabbitmq" -> queueTypeName = "RabbitMQ"; |
|||
default -> { |
|||
return; |
|||
} |
|||
} |
|||
|
|||
log.warn("WARNING: {} queue type is deprecated and will be removed in ThingsBoard 4.0. Please migrate to Apache Kafka", queueTypeName); |
|||
notificationCenter.sendGeneralWebNotification(TenantId.SYS_TENANT_ID, new SystemAdministratorsFilter(), |
|||
DefaultNotifications.queueTypeDeprecation.toTemplate(), new GeneralNotificationInfo(Map.of( |
|||
"queueType", queueTypeName |
|||
))); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,74 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.entitiy.dashboard; |
|||
|
|||
import org.junit.After; |
|||
import org.junit.Test; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.mock.web.MockHttpServletResponse; |
|||
import org.springframework.test.context.TestPropertySource; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.server.common.data.Dashboard; |
|||
import org.thingsboard.server.controller.AbstractControllerTest; |
|||
import org.thingsboard.server.dao.service.DaoSqlTest; |
|||
import org.thingsboard.server.dao.sql.resource.TbResourceRepository; |
|||
import org.thingsboard.server.dao.sql.widget.WidgetTypeRepository; |
|||
import org.thingsboard.server.dao.sql.widget.WidgetsBundleRepository; |
|||
|
|||
import java.util.concurrent.TimeUnit; |
|||
|
|||
import static org.assertj.core.api.Assertions.assertThat; |
|||
import static org.awaitility.Awaitility.await; |
|||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; |
|||
|
|||
@DaoSqlTest |
|||
@TestPropertySource(properties = { |
|||
"transport.gateway.dashboard.sync.enabled=true" |
|||
}) |
|||
public class DashboardSyncServiceTest extends AbstractControllerTest { |
|||
|
|||
@Autowired |
|||
private WidgetTypeRepository widgetTypeRepository; |
|||
@Autowired |
|||
private WidgetsBundleRepository widgetsBundleRepository; |
|||
@Autowired |
|||
private TbResourceRepository resourceRepository; |
|||
|
|||
@After |
|||
public void after() throws Exception { |
|||
widgetsBundleRepository.deleteAll(); |
|||
widgetTypeRepository.deleteAll(); |
|||
resourceRepository.deleteAll(); |
|||
} |
|||
|
|||
@Test |
|||
public void testGatewaysDashboardSync() throws Exception { |
|||
loginTenantAdmin(); |
|||
await().atMost(60, TimeUnit.SECONDS).untilAsserted(() -> { |
|||
MockHttpServletResponse response = doGet("/api/resource/dashboard/system/gateways_dashboard.json") |
|||
.andExpect(status().isOk()) |
|||
.andReturn().getResponse(); |
|||
String dashboardJson = response.getContentAsString(); |
|||
String etag = response.getHeader("ETag"); |
|||
|
|||
Dashboard dashboard = JacksonUtil.fromString(dashboardJson, Dashboard.class); |
|||
assertThat(dashboard).isNotNull(); |
|||
assertThat(dashboard.getTitle()).containsIgnoringCase("gateway"); |
|||
assertThat(etag).isNotBlank(); |
|||
}); |
|||
} |
|||
|
|||
} |
|||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue