public class FederationInterceptor extends AbstractRequestInterceptor
| Modifier and Type | Field and Description |
|---|---|
static String |
NMSS_CLASS_PREFIX |
static String |
NMSS_REG_REQUEST_KEY |
static String |
NMSS_REG_RESPONSE_KEY |
static String |
NMSS_SECONDARY_SC_PREFIX
When AMRMProxy HA is enabled, secondary AMRMTokens will be stored in Yarn
Registry.
|
static String |
STRING_TO_BYTE_FORMAT |
| Constructor and Description |
|---|
FederationInterceptor()
Creates an instance of the FederationInterceptor class.
|
| Modifier and Type | Method and Description |
|---|---|
org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse |
allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest request)
Sends the heart beats to the home RM and the secondary sub-cluster RMs that
are being used by the application.
|
protected void |
cacheAllocatedContainersForSubClusterId(List<org.apache.hadoop.yarn.api.records.Container> containers,
org.apache.hadoop.yarn.server.federation.store.records.SubClusterId subClusterId) |
protected void |
cleanupRegistry()
Only for unit test cleanup.
|
protected org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler |
createHomeHeartbeatHandler(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.yarn.api.records.ApplicationId appId,
org.apache.hadoop.yarn.server.AMRMClientRelayer rmProxyRelayer) |
protected <T> T |
createHomeRMProxy(AMRMProxyApplicationContext appContext,
Class<T> protocol,
org.apache.hadoop.security.UserGroupInformation user)
Create a proxy instance that is used to connect to the Home resource
manager.
|
protected org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager |
createUnmanagedAMPoolManager(ExecutorService threadPool)
Create the UAM pool manager for secondary sub-clusters.
|
org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse |
finishApplicationMaster(org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest request)
Sends the finish application master request to all the resource managers
used by the application.
|
protected org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse |
generateBaseAllocationResponse()
Prepare the base allocation response.
|
Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,List<org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse>> |
getAsyncResponseSink() |
protected org.apache.hadoop.yarn.api.records.ApplicationAttemptId |
getAttemptId() |
protected Map<org.apache.hadoop.yarn.api.records.ContainerId,org.apache.hadoop.yarn.server.federation.store.records.SubClusterId> |
getContainerIdToSubClusterIdMap() |
protected org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler |
getHomeHeartbeatHandler() |
protected org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient |
getRegistryClient() |
protected Set<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId> |
getTimedOutSCs(boolean verbose) |
protected Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,Future<?>> |
getUamRegisterFutures() |
protected org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager |
getUnmanagedAMPool() |
protected int |
getUnmanagedAMPoolSize() |
void |
init(AMRMProxyApplicationContext appContext)
Initializes the instance using specified context.
|
static <T> boolean |
isNullOrEmpty(Collection<T> c)
Utility method to check if the specified Collection is null or empty.
|
static <T1,T2> boolean |
isNullOrEmpty(Map<T1,T2> c)
Utility method to check if the specified Collection is null or empty.
|
protected TokenAndRegisterResponse |
launchUAMAndRegisterApplicationMaster(org.apache.hadoop.yarn.conf.YarnConfiguration config,
String subClusterId,
org.apache.hadoop.yarn.api.records.ApplicationId applicationId) |
protected void |
mergeAllocateResponse(org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse homeResponse,
org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse otherResponse,
org.apache.hadoop.yarn.server.federation.store.records.SubClusterId otherRMAddress) |
protected void |
reAttachUAMAndMergeRegisterResponse(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse homeResponse,
org.apache.hadoop.yarn.api.records.ApplicationId appId)
Try re-attach to all existing and running UAMs in secondary sub-clusters
launched by previous application attempts if any.
|
void |
recover(Map<String,byte[]> recoveredDataMap)
Recover
RequestInterceptor state from store. |
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse |
registerApplicationMaster(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest request)
Sends the application master's registration request to the home RM.
|
void |
setNextInterceptor(RequestInterceptor next)
Sets the
RequestInterceptor in the chain. |
void |
shutdown()
This is called when the application pipeline is being destroyed.
|
protected Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,List<org.apache.hadoop.yarn.api.records.ResourceRequest>> |
splitResourceRequests(List<org.apache.hadoop.yarn.api.records.ResourceRequest> askList)
Splits the specified request to send it to different sub clusters.
|
allocateForDistributedScheduling, getApplicationContext, getConf, getNextInterceptor, getNMStateStore, registerApplicationMasterForDistributedScheduling, setConfpublic static final String NMSS_CLASS_PREFIX
public static final String NMSS_REG_REQUEST_KEY
public static final String NMSS_REG_RESPONSE_KEY
public static final String NMSS_SECONDARY_SC_PREFIX
public static final String STRING_TO_BYTE_FORMAT
public FederationInterceptor()
public void init(AMRMProxyApplicationContext appContext)
init in interface RequestInterceptorinit in class AbstractRequestInterceptorappContext - AMRMProxy application contextpublic void recover(Map<String,byte[]> recoveredDataMap)
AbstractRequestInterceptorRequestInterceptor state from store.recover in interface RequestInterceptorrecover in class AbstractRequestInterceptorrecoveredDataMap - states for all interceptors recovered from NMSSpublic org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse registerApplicationMaster(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest request)
throws org.apache.hadoop.yarn.exceptions.YarnException,
IOException
InvalidApplicationMasterRequestException. Furthermore, we present
to AM as if we are the RM that never fails over (except when AMRMProxy
restarts). When actual RM fails over, we always re-register automatically.
We did this because FederationInterceptor can receive concurrent register
requests from AM because of timeout between AM and AMRMProxy, which is
shorter than the timeout + failOver between FederationInterceptor
(AMRMProxy) and RM.
For the same reason, this method needs to be synchronized.org.apache.hadoop.yarn.exceptions.YarnExceptionIOExceptionpublic org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest request)
throws org.apache.hadoop.yarn.exceptions.YarnException,
IOException
org.apache.hadoop.yarn.exceptions.YarnExceptionIOExceptionpublic org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse finishApplicationMaster(org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest request)
throws org.apache.hadoop.yarn.exceptions.YarnException,
IOException
org.apache.hadoop.yarn.exceptions.YarnExceptionIOExceptionpublic void setNextInterceptor(RequestInterceptor next)
AbstractRequestInterceptorRequestInterceptor in the chain.setNextInterceptor in interface RequestInterceptorsetNextInterceptor in class AbstractRequestInterceptornext - the next interceptor to setpublic void shutdown()
shutdown in interface RequestInterceptorshutdown in class AbstractRequestInterceptor@VisibleForTesting protected void cleanupRegistry()
@VisibleForTesting protected org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient getRegistryClient()
@VisibleForTesting protected org.apache.hadoop.yarn.api.records.ApplicationAttemptId getAttemptId()
@VisibleForTesting protected org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler getHomeHeartbeatHandler()
@VisibleForTesting protected org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager createUnmanagedAMPoolManager(ExecutorService threadPool)
threadPool - the thread pool to use@VisibleForTesting
protected org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler createHomeHeartbeatHandler(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.yarn.api.records.ApplicationId appId,
org.apache.hadoop.yarn.server.AMRMClientRelayer rmProxyRelayer)
protected <T> T createHomeRMProxy(AMRMProxyApplicationContext appContext, Class<T> protocol, org.apache.hadoop.security.UserGroupInformation user)
T - the type of the proxyappContext - AMRMProxyApplicationContextprotocol - the protocol class for the proxyuser - the ugi for the proxyprotected void reAttachUAMAndMergeRegisterResponse(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse homeResponse,
org.apache.hadoop.yarn.api.records.ApplicationId appId)
protected TokenAndRegisterResponse launchUAMAndRegisterApplicationMaster(org.apache.hadoop.yarn.conf.YarnConfiguration config, String subClusterId, org.apache.hadoop.yarn.api.records.ApplicationId applicationId) throws IOException, org.apache.hadoop.yarn.exceptions.YarnException
IOExceptionorg.apache.hadoop.yarn.exceptions.YarnExceptionprotected org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse generateBaseAllocationResponse()
@VisibleForTesting
protected void mergeAllocateResponse(org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse homeResponse,
org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse otherResponse,
org.apache.hadoop.yarn.server.federation.store.records.SubClusterId otherRMAddress)
protected Set<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId> getTimedOutSCs(boolean verbose)
protected Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,List<org.apache.hadoop.yarn.api.records.ResourceRequest>> splitResourceRequests(List<org.apache.hadoop.yarn.api.records.ResourceRequest> askList) throws org.apache.hadoop.yarn.exceptions.YarnException
askList - the ask list to splitorg.apache.hadoop.yarn.exceptions.YarnException - if split fails@VisibleForTesting protected int getUnmanagedAMPoolSize()
@VisibleForTesting protected org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager getUnmanagedAMPool()
@VisibleForTesting protected Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,Future<?>> getUamRegisterFutures()
@VisibleForTesting public Map<org.apache.hadoop.yarn.server.federation.store.records.SubClusterId,List<org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse>> getAsyncResponseSink()
public static <T> boolean isNullOrEmpty(Collection<T> c)
T - element type of the collectionc - the collection objectpublic static <T1,T2> boolean isNullOrEmpty(Map<T1,T2> c)
T1 - key type of the mapT2 - value type of the mapc - the map object@VisibleForTesting protected void cacheAllocatedContainersForSubClusterId(List<org.apache.hadoop.yarn.api.records.Container> containers, org.apache.hadoop.yarn.server.federation.store.records.SubClusterId subClusterId)
@VisibleForTesting protected Map<org.apache.hadoop.yarn.api.records.ContainerId,org.apache.hadoop.yarn.server.federation.store.records.SubClusterId> getContainerIdToSubClusterIdMap()
Copyright © 2008–2024 Apache Software Foundation. All rights reserved.