<?xml version="1.0" encoding="UTF-8"?>

<!--XBRL Document Created with Wdesk from Workiva-->
<!--Copyright 2021 Workiva-->
<!--r:1321cc64-8719-4b33-b37d-5116d8a2118a,g:7b53f3f1-5374-4423-b06e-297ed9632f8b-->
<link:linkbase xmlns:link="http://www.xbrl.org/2003/linkbase" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd">
  <link:roleRef roleURI="http://www.xbrl.org/2009/role/negatedPeriodStartLabel" xlink:type="simple" xlink:href="http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#negatedPeriodStartLabel"/>
  <link:roleRef roleURI="http://www.xbrl.org/2009/role/netLabel" xlink:type="simple" xlink:href="http://www.xbrl.org/lrr/role/net-2009-12-16.xsd#netLabel"/>
  <link:roleRef roleURI="http://www.xbrl.org/2009/role/negatedNetLabel" xlink:type="simple" xlink:href="http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#negatedNetLabel"/>
  <link:roleRef roleURI="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="simple" xlink:href="http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#negatedTerseLabel"/>
  <link:roleRef roleURI="http://www.xbrl.org/2009/role/negatedPeriodEndLabel" xlink:type="simple" xlink:href="http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#negatedPeriodEndLabel"/>
  <link:roleRef roleURI="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="simple" xlink:href="http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#negatedLabel"/>
  <link:roleRef roleURI="http://www.xbrl.org/2009/role/negatedTotalLabel" xlink:type="simple" xlink:href="http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#negatedTotalLabel"/>
  <link:labelLink xlink:role="http://www.xbrl.org/2003/role/link" xlink:type="extended">
    <link:label id="lab_us-gaap_PaymentsForDerivativeInstrumentInvestingActivities_f6847059-d07c-4c3b-b1ef-ffe5fbb4a1f7_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_PaymentsForDerivativeInstrumentInvestingActivities" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments on financial derivatives</link:label>
    <link:label id="lab_us-gaap_PaymentsForDerivativeInstrumentInvestingActivities_label_en-US" xlink:label="lab_us-gaap_PaymentsForDerivativeInstrumentInvestingActivities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments for Derivative Instrument, Investing Activities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PaymentsForDerivativeInstrumentInvestingActivities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsForDerivativeInstrumentInvestingActivities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PaymentsForDerivativeInstrumentInvestingActivities" xlink:to="lab_us-gaap_PaymentsForDerivativeInstrumentInvestingActivities" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansBetween500000And749999Member_ff494741-ce9c-44ca-9ad7-5f5ccf42384f_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansBetween500000And749999Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $500,000 and $749,999</link:label>
    <link:label id="lab_srt_MortgageLoansBetween500000And749999Member_label_en-US" xlink:label="lab_srt_MortgageLoansBetween500000And749999Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $500,000 and $749,999 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansBetween500000And749999Member" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansBetween500000And749999Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansBetween500000And749999Member" xlink:to="lab_srt_MortgageLoansBetween500000And749999Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleOfPercentageOfTotalReceivableForSecuritiesSoldTableTextBlock_1ec99b76-bf86-4723-814c-9008c3ce5072_terseLabel_en-US" xlink:label="lab_efc_ScheduleOfPercentageOfTotalReceivableForSecuritiesSoldTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">% Of Total Receivable For Securities Sold</link:label>
    <link:label id="lab_efc_ScheduleOfPercentageOfTotalReceivableForSecuritiesSoldTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleOfPercentageOfTotalReceivableForSecuritiesSoldTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Percentage Of Total Receivable For Securities Sold [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleOfPercentageOfTotalReceivableForSecuritiesSoldTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleOfPercentageOfTotalReceivableForSecuritiesSoldTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Tabular disclosure of the Company's percentage of total amount receivable for securities sold held by counterparties above a specified concentration risk threshold.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleOfPercentageOfTotalReceivableForSecuritiesSoldTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleOfPercentageOfTotalReceivableForSecuritiesSoldTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleOfPercentageOfTotalReceivableForSecuritiesSoldTableTextBlock" xlink:to="lab_efc_ScheduleOfPercentageOfTotalReceivableForSecuritiesSoldTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FinancingReceivablesUnpaidPrincipalBalanceMember_b7569579-3380-4a8b-a501-a5ffce53c08a_terseLabel_en-US" xlink:label="lab_efc_FinancingReceivablesUnpaidPrincipalBalanceMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivables, Unpaid Principal Balance [Member]</link:label>
    <link:label id="lab_efc_FinancingReceivablesUnpaidPrincipalBalanceMember_label_en-US" xlink:label="lab_efc_FinancingReceivablesUnpaidPrincipalBalanceMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivables, Unpaid Principal Balance [Member]</link:label>
    <link:label id="lab_efc_FinancingReceivablesUnpaidPrincipalBalanceMember_documentation_en-US" xlink:label="lab_efc_FinancingReceivablesUnpaidPrincipalBalanceMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivables, Unpaid Principal Balance [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FinancingReceivablesUnpaidPrincipalBalanceMember" xlink:href="efc-20201231.xsd#efc_FinancingReceivablesUnpaidPrincipalBalanceMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FinancingReceivablesUnpaidPrincipalBalanceMember" xlink:to="lab_efc_FinancingReceivablesUnpaidPrincipalBalanceMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest_ee606029-7f12-440e-ac65-e9ab9e1ffb5b_totalLabel_en-US" xlink:label="lab_us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Equity</link:label>
    <link:label id="lab_us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest_7b65cb5a-14a3-4262-9496-87db263212a6_periodStartLabel_en-US" xlink:label="lab_us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Beginning balance</link:label>
    <link:label id="lab_us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest_e32bc117-fbe5-43e9-b4ed-33b13be1a199_periodEndLabel_en-US" xlink:label="lab_us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ending balance</link:label>
    <link:label id="lab_us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest_ba8de408-9c99-4235-8046-2f7dc150ae76_terseLabel_en-US" xlink:label="lab_us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Equity</link:label>
    <link:label id="lab_us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest_label_en-US" xlink:label="lab_us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stockholders' Equity, Including Portion Attributable to Noncontrolling Interest</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" xlink:to="lab_us-gaap_StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CurrencyriskMember_f24e0d25-619b-4d53-b45b-6c7eed9547df_terseLabel_en-US" xlink:label="lab_efc_CurrencyriskMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Currency risk [Member]</link:label>
    <link:label id="lab_efc_CurrencyriskMember_label_en-US" xlink:label="lab_efc_CurrencyriskMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Currency risk [Member]</link:label>
    <link:label id="lab_efc_CurrencyriskMember_documentation_en-US" xlink:label="lab_efc_CurrencyriskMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Currency risk [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CurrencyriskMember" xlink:href="efc-20201231.xsd#efc_CurrencyriskMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CurrencyriskMember" xlink:to="lab_efc_CurrencyriskMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationByLiabilityClassDomain_e55cd444-038e-4f98-9ec3-61524bd69053_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationByLiabilityClassDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value by Liability Class [Domain]</link:label>
    <link:label id="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationByLiabilityClassDomain_label_en-US" xlink:label="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationByLiabilityClassDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value by Liability Class [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationByLiabilityClassDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationByLiabilityClassDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationByLiabilityClassDomain" xlink:to="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationByLiabilityClassDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RelatedpartymortgageoriginatorwithflowmortgageloanpurchaseandsaleagreementMember_d0a3bbb5-30a9-417b-8bdb-459f4368ee8d_terseLabel_en-US" xlink:label="lab_efc_RelatedpartymortgageoriginatorwithflowmortgageloanpurchaseandsaleagreementMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related party mortgage originator (with flow mortgage loan purchase and sale agreement) [Member]</link:label>
    <link:label id="lab_efc_RelatedpartymortgageoriginatorwithflowmortgageloanpurchaseandsaleagreementMember_label_en-US" xlink:label="lab_efc_RelatedpartymortgageoriginatorwithflowmortgageloanpurchaseandsaleagreementMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related party mortgage originator (with flow mortgage loan purchase and sale agreement) [Member]</link:label>
    <link:label id="lab_efc_RelatedpartymortgageoriginatorwithflowmortgageloanpurchaseandsaleagreementMember_documentation_en-US" xlink:label="lab_efc_RelatedpartymortgageoriginatorwithflowmortgageloanpurchaseandsaleagreementMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related party mortgage originator (with flow mortgage loan purchase and sale agreement) [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RelatedpartymortgageoriginatorwithflowmortgageloanpurchaseandsaleagreementMember" xlink:href="efc-20201231.xsd#efc_RelatedpartymortgageoriginatorwithflowmortgageloanpurchaseandsaleagreementMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RelatedpartymortgageoriginatorwithflowmortgageloanpurchaseandsaleagreementMember" xlink:to="lab_efc_RelatedpartymortgageoriginatorwithflowmortgageloanpurchaseandsaleagreementMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Percentageusedtocalculateservicingadministratorfee_303dec77-854a-4401-b810-ae6bd9529569_terseLabel_en-US" xlink:label="lab_efc_Percentageusedtocalculateservicingadministratorfee" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage used to calculate servicing administrator fee</link:label>
    <link:label id="lab_efc_Percentageusedtocalculateservicingadministratorfee_label_en-US" xlink:label="lab_efc_Percentageusedtocalculateservicingadministratorfee" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage used to calculate servicing administrator fee</link:label>
    <link:label id="lab_efc_Percentageusedtocalculateservicingadministratorfee_documentation_en-US" xlink:label="lab_efc_Percentageusedtocalculateservicingadministratorfee" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage used in calculating the servicing administrator fee the Company receives in connection with a securitization transaction.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Percentageusedtocalculateservicingadministratorfee" xlink:href="efc-20201231.xsd#efc_Percentageusedtocalculateservicingadministratorfee"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Percentageusedtocalculateservicingadministratorfee" xlink:to="lab_efc_Percentageusedtocalculateservicingadministratorfee" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueInputsLevel3Member_e2434d75-ce0f-4efd-8597-9a77db286ae1_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueInputsLevel3Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Level 3</link:label>
    <link:label id="lab_us-gaap_FairValueInputsLevel3Member_label_en-US" xlink:label="lab_us-gaap_FairValueInputsLevel3Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Inputs, Level 3 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueInputsLevel3Member" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueInputsLevel3Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueInputsLevel3Member" xlink:to="lab_us-gaap_FairValueInputsLevel3Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearFourPercentMember_b0f14a4f-6203-45ca-ac03-e1c04a1abeb7_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Fifteen Year) Four Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearFourPercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Fifteen Year) Four Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearFourPercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Fifteen Year) Four Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearFourPercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearFourPercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearFourPercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearFourPercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionFairValueTableTextBlock_7d217aa9-e014-4e8e-ad34-da9628f658c8_terseLabel_en-US" xlink:label="lab_us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionFairValueTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Unrealized Loss Position, Fair Value</link:label>
    <link:label id="lab_us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionFairValueTableTextBlock_label_en-US" xlink:label="lab_us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionFairValueTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Available-for-sale Securities, Continuous Unrealized Loss Position, Fair Value [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionFairValueTableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionFairValueTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionFairValueTableTextBlock" xlink:to="lab_us-gaap_AvailableForSaleSecuritiesContinuousUnrealizedLossPositionFairValueTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year5Point50PercentMember_559609c4-c833-45e4-9002-c05a0ca0acd8_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year5Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 5 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year5Point50PercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year5Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 5 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year5Point50PercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year5Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 5.50%</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociation30Year5Point50PercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociation30Year5Point50PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociation30Year5Point50PercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociation30Year5Point50PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AdjustmentsToAdditionalPaidInCapitalSharebasedCompensationRequisiteServicePeriodRecognitionValue_8c3bc334-4a76-4745-9820-f032a7886024_terseLabel_en-US" xlink:label="lab_us-gaap_AdjustmentsToAdditionalPaidInCapitalSharebasedCompensationRequisiteServicePeriodRecognitionValue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based long term incentive plan unit awards</link:label>
    <link:label id="lab_us-gaap_AdjustmentsToAdditionalPaidInCapitalSharebasedCompensationRequisiteServicePeriodRecognitionValue_label_en-US" xlink:label="lab_us-gaap_AdjustmentsToAdditionalPaidInCapitalSharebasedCompensationRequisiteServicePeriodRecognitionValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">APIC, Share-based Payment Arrangement, Increase for Cost Recognition</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AdjustmentsToAdditionalPaidInCapitalSharebasedCompensationRequisiteServicePeriodRecognitionValue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AdjustmentsToAdditionalPaidInCapitalSharebasedCompensationRequisiteServicePeriodRecognitionValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AdjustmentsToAdditionalPaidInCapitalSharebasedCompensationRequisiteServicePeriodRecognitionValue" xlink:to="lab_us-gaap_AdjustmentsToAdditionalPaidInCapitalSharebasedCompensationRequisiteServicePeriodRecognitionValue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202808Member_62d5e030-49c9-48d2-84f2-b6bb27500f9f_terseLabel_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202808Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.88 Percent, Maturity Date 2028 08 [Member]</link:label>
    <link:label id="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202808Member_label_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202808Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.88 Percent, Maturity Date 2028 08 [Member]</link:label>
    <link:label id="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202808Member_documentation_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202808Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.88 Percent, Maturity Date 2028 08 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_U.S.TreasuryNote2.88PercentMaturityDate202808Member" xlink:href="efc-20201231.xsd#efc_U.S.TreasuryNote2.88PercentMaturityDate202808Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_U.S.TreasuryNote2.88PercentMaturityDate202808Member" xlink:to="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202808Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Investmentinloans_df9eafe2-fff2-4231-b708-6cdd5410c6f2_terseLabel_en-US" xlink:label="lab_efc_Investmentinloans" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans, at fair value</link:label>
    <link:label id="lab_efc_Investmentinloans_bc4ad0e3-f905-4377-8b6d-6445090e0de3_verboseLabel_en-US" xlink:label="lab_efc_Investmentinloans" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value</link:label>
    <link:label id="lab_efc_Investmentinloans_6fe4988c-2200-4077-8991-9e16768fe12c_totalLabel_en-US" xlink:label="lab_efc_Investmentinloans" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value</link:label>
    <link:label id="lab_efc_Investmentinloans_label_en-US" xlink:label="lab_efc_Investmentinloans" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment in loans</link:label>
    <link:label id="lab_efc_Investmentinloans_documentation_en-US" xlink:label="lab_efc_Investmentinloans" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair value of loans which can include loans held-for-investment or held-for-sale.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Investmentinloans" xlink:href="efc-20201231.xsd#efc_Investmentinloans"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Investmentinloans" xlink:to="lab_efc_Investmentinloans" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ExpenseReimbursementPeriod_b7d4d25c-8f1a-4bcb-96e6-df26ad83d264_terseLabel_en-US" xlink:label="lab_efc_ExpenseReimbursementPeriod" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Expense Reimbursement Period</link:label>
    <link:label id="lab_efc_ExpenseReimbursementPeriod_label_en-US" xlink:label="lab_efc_ExpenseReimbursementPeriod" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Expense Reimbursement Period</link:label>
    <link:label id="lab_efc_ExpenseReimbursementPeriod_documentation_en-US" xlink:label="lab_efc_ExpenseReimbursementPeriod" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of days in which the Company will reimburse the Manager following the receipt of an expense invoice.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ExpenseReimbursementPeriod" xlink:href="efc-20201231.xsd#efc_ExpenseReimbursementPeriod"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ExpenseReimbursementPeriod" xlink:to="lab_efc_ExpenseReimbursementPeriod" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DisclosureOfCreditDerivativesTextBlock_6d239bb3-c799-4086-abf1-8601cd34ba60_terseLabel_en-US" xlink:label="lab_us-gaap_DisclosureOfCreditDerivativesTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Credit Derivatives</link:label>
    <link:label id="lab_us-gaap_DisclosureOfCreditDerivativesTextBlock_label_en-US" xlink:label="lab_us-gaap_DisclosureOfCreditDerivativesTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disclosure of Credit Derivatives [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DisclosureOfCreditDerivativesTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DisclosureOfCreditDerivativesTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DisclosureOfCreditDerivativesTextBlock" xlink:to="lab_us-gaap_DisclosureOfCreditDerivativesTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MortgageLoanOriginatorMember_ab2f22f5-6f90-4664-b449-db2f22767614_terseLabel_en-US" xlink:label="lab_efc_MortgageLoanOriginatorMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage loan originator [Member]</link:label>
    <link:label id="lab_efc_MortgageLoanOriginatorMember_label_en-US" xlink:label="lab_efc_MortgageLoanOriginatorMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage loan originator [Member]</link:label>
    <link:label id="lab_efc_MortgageLoanOriginatorMember_documentation_en-US" xlink:label="lab_efc_MortgageLoanOriginatorMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage loan originator</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MortgageLoanOriginatorMember" xlink:href="efc-20201231.xsd#efc_MortgageLoanOriginatorMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MortgageLoanOriginatorMember" xlink:to="lab_efc_MortgageLoanOriginatorMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AccruedLiabilitiesAndOtherLiabilities_05519caf-64a9-4725-a156-ee7f5f83856b_terseLabel_en-US" xlink:label="lab_us-gaap_AccruedLiabilitiesAndOtherLiabilities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accrued expenses and other liabilities</link:label>
    <link:label id="lab_us-gaap_AccruedLiabilitiesAndOtherLiabilities_label_en-US" xlink:label="lab_us-gaap_AccruedLiabilitiesAndOtherLiabilities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accrued Liabilities and Other Liabilities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AccruedLiabilitiesAndOtherLiabilities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AccruedLiabilitiesAndOtherLiabilities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AccruedLiabilitiesAndOtherLiabilities" xlink:to="lab_us-gaap_AccruedLiabilitiesAndOtherLiabilities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NoncontrollinginterestinOperatingPartnershipMember_b7992b3d-f5c8-415f-ac9f-1f369152a8ac_terseLabel_en-US" xlink:label="lab_efc_NoncontrollinginterestinOperatingPartnershipMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling interest in Operating Partnership [Member]</link:label>
    <link:label id="lab_efc_NoncontrollinginterestinOperatingPartnershipMember_label_en-US" xlink:label="lab_efc_NoncontrollinginterestinOperatingPartnershipMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling interest in Operating Partnership [Member]</link:label>
    <link:label id="lab_efc_NoncontrollinginterestinOperatingPartnershipMember_documentation_en-US" xlink:label="lab_efc_NoncontrollinginterestinOperatingPartnershipMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling interest in Operating Partnership. Non-controlling interest issued operating partnership units which are convertible into common shares of the Company.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NoncontrollinginterestinOperatingPartnershipMember" xlink:href="efc-20201231.xsd#efc_NoncontrollinginterestinOperatingPartnershipMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NoncontrollinginterestinOperatingPartnershipMember" xlink:to="lab_efc_NoncontrollinginterestinOperatingPartnershipMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Grossunrealizedgains_61670070-cc13-444a-986a-23ec09949cb1_terseLabel_en-US" xlink:label="lab_efc_Grossunrealizedgains" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Gains</link:label>
    <link:label id="lab_efc_Grossunrealizedgains_label_en-US" xlink:label="lab_efc_Grossunrealizedgains" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Gross unrealized gains</link:label>
    <link:label id="lab_efc_Grossunrealizedgains_documentation_en-US" xlink:label="lab_efc_Grossunrealizedgains" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Gross unrealized gain on investment.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Grossunrealizedgains" xlink:href="efc-20201231.xsd#efc_Grossunrealizedgains"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Grossunrealizedgains" xlink:to="lab_efc_Grossunrealizedgains" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FairValueofAgencyIOsnegativeOASmeasurementinput_1e59c190-1280-477a-aab3-55b09823c7e4_terseLabel_en-US" xlink:label="lab_efc_FairValueofAgencyIOsnegativeOASmeasurementinput" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value of Agency IOs, negative OAS measurement input</link:label>
    <link:label id="lab_efc_FairValueofAgencyIOsnegativeOASmeasurementinput_label_en-US" xlink:label="lab_efc_FairValueofAgencyIOsnegativeOASmeasurementinput" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value of Agency IOs, negative OAS measurement input</link:label>
    <link:label id="lab_efc_FairValueofAgencyIOsnegativeOASmeasurementinput_documentation_en-US" xlink:label="lab_efc_FairValueofAgencyIOsnegativeOASmeasurementinput" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair value of Agency IOs that have negative OAS measurement unputs at valuation date.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FairValueofAgencyIOsnegativeOASmeasurementinput" xlink:href="efc-20201231.xsd#efc_FairValueofAgencyIOsnegativeOASmeasurementinput"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FairValueofAgencyIOsnegativeOASmeasurementinput" xlink:to="lab_efc_FairValueofAgencyIOsnegativeOASmeasurementinput" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueByAssetClassAxis_3c564b33-095e-4008-92be-06aa7ef243d6_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueByAssetClassAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Asset Class [Axis]</link:label>
    <link:label id="lab_us-gaap_FairValueByAssetClassAxis_label_en-US" xlink:label="lab_us-gaap_FairValueByAssetClassAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Asset Class [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueByAssetClassAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueByAssetClassAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueByAssetClassAxis" xlink:to="lab_us-gaap_FairValueByAssetClassAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SocieteGeneraleCollateralizedbyParValue100ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member_031713d2-9a04-48bb-92f5-99b01a0c20a7_terseLabel_en-US" xlink:label="lab_efc_SocieteGeneraleCollateralizedbyParValue100ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Societe Generale Collateralized by Par Value $100 Exchange-Traded Corporate Debt, Coupon 5.95%, Maturity Date 2026 12 [Member]</link:label>
    <link:label id="lab_efc_SocieteGeneraleCollateralizedbyParValue100ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member_label_en-US" xlink:label="lab_efc_SocieteGeneraleCollateralizedbyParValue100ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Societe Generale Collateralized by Par Value $100 Exchange-Traded Corporate Debt, Coupon 5.95%, Maturity Date 2026 12 [Member]</link:label>
    <link:label id="lab_efc_SocieteGeneraleCollateralizedbyParValue100ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member_documentation_en-US" xlink:label="lab_efc_SocieteGeneraleCollateralizedbyParValue100ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Societe Generale Collateralized by Par Value $100 Exchange-Traded Corporate Debt, Coupon 5.95%, Maturity Date 2026 12 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SocieteGeneraleCollateralizedbyParValue100ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:href="efc-20201231.xsd#efc_SocieteGeneraleCollateralizedbyParValue100ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SocieteGeneraleCollateralizedbyParValue100ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:to="lab_efc_SocieteGeneraleCollateralizedbyParValue100ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3250000And3499000Member_0e8c67f8-000b-4e8c-9ff2-3f09d5cc56aa_terseLabel_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3250000And3499000Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $3,250,000 and $3,499,000</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3250000And3499000Member_label_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3250000And3499000Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $3,250,000 and $3,499,000 [Member]</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3250000And3499000Member_documentation_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3250000And3499000Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $3,250,000 and $3,499,000</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3250000And3499000Member" xlink:href="efc-20201231.xsd#efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3250000And3499000Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3250000And3499000Member" xlink:to="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3250000And3499000Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DerivativeCostBasis_de42b219-3d0e-49f9-8212-2bdc3fe15d37_terseLabel_en-US" xlink:label="lab_efc_DerivativeCostBasis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative, Cost Basis</link:label>
    <link:label id="lab_efc_DerivativeCostBasis_label_en-US" xlink:label="lab_efc_DerivativeCostBasis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative, Cost Basis</link:label>
    <link:label id="lab_efc_DerivativeCostBasis_documentation_en-US" xlink:label="lab_efc_DerivativeCostBasis" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative, Cost Basis</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DerivativeCostBasis" xlink:href="efc-20201231.xsd#efc_DerivativeCostBasis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DerivativeCostBasis" xlink:to="lab_efc_DerivativeCostBasis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseMember_0688fe83-619c-45f8-a132-e932ccde3b9b_verboseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseMember" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase Agreements</link:label>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseMember_f1af2d3b-c6fd-4eba-b422-b8c607f9e2d0_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reverse repurchase agreements</link:label>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseMember_label_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Sold under Agreements to Repurchase [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseMember" xlink:to="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseMember" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_CurrentFiscalYearEndDate_024f32a7-2a93-466e-b4f2-d78a5dfd871d_terseLabel_en-US" xlink:label="lab_dei_CurrentFiscalYearEndDate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Current Fiscal Year End Date</link:label>
    <link:label id="lab_dei_CurrentFiscalYearEndDate_label_en-US" xlink:label="lab_dei_CurrentFiscalYearEndDate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Current Fiscal Year End Date</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_CurrentFiscalYearEndDate" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_CurrentFiscalYearEndDate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_CurrentFiscalYearEndDate" xlink:to="lab_dei_CurrentFiscalYearEndDate" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PaymentsToBrokersFinancingActivities_7fb79d91-8802-4638-88b3-2accd6ff7966_negatedTerseLabel_en-US" xlink:label="lab_efc_PaymentsToBrokersFinancingActivities" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Due to brokers, net</link:label>
    <link:label id="lab_efc_PaymentsToBrokersFinancingActivities_label_en-US" xlink:label="lab_efc_PaymentsToBrokersFinancingActivities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments To Brokers, Financing Activities</link:label>
    <link:label id="lab_efc_PaymentsToBrokersFinancingActivities_documentation_en-US" xlink:label="lab_efc_PaymentsToBrokersFinancingActivities" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments To Brokers, Financing Activities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PaymentsToBrokersFinancingActivities" xlink:href="efc-20201231.xsd#efc_PaymentsToBrokersFinancingActivities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PaymentsToBrokersFinancingActivities" xlink:to="lab_efc_PaymentsToBrokersFinancingActivities" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProceedsFromIssuanceOfCommonStock_72c9e30f-74a3-4da3-8949-3b551aa0a0af_terseLabel_en-US" xlink:label="lab_us-gaap_ProceedsFromIssuanceOfCommonStock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net proceeds from issuance of common stock</link:label>
    <link:label id="lab_us-gaap_ProceedsFromIssuanceOfCommonStock_label_en-US" xlink:label="lab_us-gaap_ProceedsFromIssuanceOfCommonStock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from Issuance of Common Stock</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProceedsFromIssuanceOfCommonStock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromIssuanceOfCommonStock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProceedsFromIssuanceOfCommonStock" xlink:to="lab_us-gaap_ProceedsFromIssuanceOfCommonStock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersOutOfLevel3_179a8e31-b8d2-4cb0-8035-25a94247439b_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersOutOfLevel3" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities, Transfers out of Level 3</link:label>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersOutOfLevel3_label_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersOutOfLevel3" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement with Unobservable Inputs Reconciliation, Liability, Transfers out of Level 3</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersOutOfLevel3" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersOutOfLevel3"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersOutOfLevel3" xlink:to="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersOutOfLevel3" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DividendPayableDateToBePaidDayMonthAndYear_3f09fd51-0b34-4ac8-85a2-427040d1f39d_terseLabel_en-US" xlink:label="lab_us-gaap_DividendPayableDateToBePaidDayMonthAndYear" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payment Date</link:label>
    <link:label id="lab_us-gaap_DividendPayableDateToBePaidDayMonthAndYear_label_en-US" xlink:label="lab_us-gaap_DividendPayableDateToBePaidDayMonthAndYear" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends Payable, Date to be Paid</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DividendPayableDateToBePaidDayMonthAndYear" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DividendPayableDateToBePaidDayMonthAndYear"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DividendPayableDateToBePaidDayMonthAndYear" xlink:to="lab_us-gaap_DividendPayableDateToBePaidDayMonthAndYear" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeLineItems_e11c3daa-487e-4fed-87fb-212210a6b6ea_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative [Line Items]</link:label>
    <link:label id="lab_us-gaap_DerivativeLineItems_label_en-US" xlink:label="lab_us-gaap_DerivativeLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeLineItems" xlink:to="lab_us-gaap_DerivativeLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ToBeAnnouncedFixedRateAgencySecuritiesSoldShortMember_4e5d4056-91f1-4e63-aa46-72f432aa917d_terseLabel_en-US" xlink:label="lab_efc_ToBeAnnouncedFixedRateAgencySecuritiesSoldShortMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">TBA - Fixed Rate Agency Securities Sold Short [Member]</link:label>
    <link:label id="lab_efc_ToBeAnnouncedFixedRateAgencySecuritiesSoldShortMember_label_en-US" xlink:label="lab_efc_ToBeAnnouncedFixedRateAgencySecuritiesSoldShortMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">To Be Announced Fixed Rate Agency Securities Sold Short [Member]</link:label>
    <link:label id="lab_efc_ToBeAnnouncedFixedRateAgencySecuritiesSoldShortMember_documentation_en-US" xlink:label="lab_efc_ToBeAnnouncedFixedRateAgencySecuritiesSoldShortMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">A forward settling security issued by a government agency with a fixed interest rate sold short.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ToBeAnnouncedFixedRateAgencySecuritiesSoldShortMember" xlink:href="efc-20201231.xsd#efc_ToBeAnnouncedFixedRateAgencySecuritiesSoldShortMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ToBeAnnouncedFixedRateAgencySecuritiesSoldShortMember" xlink:to="lab_efc_ToBeAnnouncedFixedRateAgencySecuritiesSoldShortMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2250000And2499999Member_20e9d082-aa6b-4437-ac48-7c932df1d6df_terseLabel_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2250000And2499999Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $2,250,000 and $2,499,999</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2250000And2499999Member_label_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2250000And2499999Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $2,250,000 and $2,499,999 [Member]</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2250000And2499999Member_documentation_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2250000And2499999Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $2,250,000 and $2,499,999</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2250000And2499999Member" xlink:href="efc-20201231.xsd#efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2250000And2499999Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2250000And2499999Member" xlink:to="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2250000And2499999Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CarryingReportedAmountFairValueDisclosureMember_52fd6813-fb15-4ddf-81a4-635d40234934_terseLabel_en-US" xlink:label="lab_us-gaap_CarryingReportedAmountFairValueDisclosureMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Carrying Value</link:label>
    <link:label id="lab_us-gaap_CarryingReportedAmountFairValueDisclosureMember_label_en-US" xlink:label="lab_us-gaap_CarryingReportedAmountFairValueDisclosureMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reported Value Measurement [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CarryingReportedAmountFairValueDisclosureMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CarryingReportedAmountFairValueDisclosureMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CarryingReportedAmountFairValueDisclosureMember" xlink:to="lab_us-gaap_CarryingReportedAmountFairValueDisclosureMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RepurchaseOfInvestmentsSoldShort_d97883c7-9db1-4c5c-a9bb-8ccf3421a72b_negatedTerseLabel_en-US" xlink:label="lab_efc_RepurchaseOfInvestmentsSoldShort" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase of investments sold short</link:label>
    <link:label id="lab_efc_RepurchaseOfInvestmentsSoldShort_label_en-US" xlink:label="lab_efc_RepurchaseOfInvestmentsSoldShort" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase of investments sold short</link:label>
    <link:label id="lab_efc_RepurchaseOfInvestmentsSoldShort_documentation_en-US" xlink:label="lab_efc_RepurchaseOfInvestmentsSoldShort" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The amount of cash outflow for the repurchase of securities to cover previous short sales.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RepurchaseOfInvestmentsSoldShort" xlink:href="efc-20201231.xsd#efc_RepurchaseOfInvestmentsSoldShort"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RepurchaseOfInvestmentsSoldShort" xlink:to="lab_efc_RepurchaseOfInvestmentsSoldShort" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Ordinaryincome_60bb0e65-4dae-4896-9a7e-27348a6bbf46_terseLabel_en-US" xlink:label="lab_efc_Ordinaryincome" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ordinary income</link:label>
    <link:label id="lab_efc_Ordinaryincome_label_en-US" xlink:label="lab_efc_Ordinaryincome" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ordinary income</link:label>
    <link:label id="lab_efc_Ordinaryincome_documentation_en-US" xlink:label="lab_efc_Ordinaryincome" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The percentage of dividends which are classified as ordinary income for tax purposes.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Ordinaryincome" xlink:href="efc-20201231.xsd#efc_Ordinaryincome"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Ordinaryincome" xlink:to="lab_efc_Ordinaryincome" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber_98216d06-3da0-4687-abcd-1f31b9cd475b_periodStartLabel_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">LTIP Units Outstanding, beginning of period</link:label>
    <link:label id="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber_919ef69e-863d-4392-82a9-386c508b00a2_periodEndLabel_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">LTIP Units Outstanding, end of period</link:label>
    <link:label id="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber_label_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Compensation Arrangement by Share-based Payment Award, Options, Outstanding, Number</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber" xlink:to="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsOutstandingNumber" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StockIssuedDuringPeriodValueNewIssues_3ce1628b-77e3-44b0-99b5-8c9559fedaf6_terseLabel_en-US" xlink:label="lab_us-gaap_StockIssuedDuringPeriodValueNewIssues" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stock Issued During Period, Value, New Issues</link:label>
    <link:label id="lab_us-gaap_StockIssuedDuringPeriodValueNewIssues_label_en-US" xlink:label="lab_us-gaap_StockIssuedDuringPeriodValueNewIssues" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stock Issued During Period, Value, New Issues</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StockIssuedDuringPeriodValueNewIssues" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockIssuedDuringPeriodValueNewIssues"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StockIssuedDuringPeriodValueNewIssues" xlink:to="lab_us-gaap_StockIssuedDuringPeriodValueNewIssues" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_VestDecember312021Member_bb8e80da-a6f8-4e65-aeb0-d25a0fdddf8f_terseLabel_en-US" xlink:label="lab_efc_VestDecember312021Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 31, 2021 [Member]</link:label>
    <link:label id="lab_efc_VestDecember312021Member_label_en-US" xlink:label="lab_efc_VestDecember312021Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 31, 2021 [Member]</link:label>
    <link:label id="lab_efc_VestDecember312021Member_documentation_en-US" xlink:label="lab_efc_VestDecember312021Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 31, 2021 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_VestDecember312021Member" xlink:href="efc-20201231.xsd#efc_VestDecember312021Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_VestDecember312021Member" xlink:to="lab_efc_VestDecember312021Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfInvestmentsTable_47002954-dc64-4b33-9943-f9ca2ea4dae9_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfInvestmentsTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Investments [Table]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfInvestmentsTable_label_en-US" xlink:label="lab_us-gaap_ScheduleOfInvestmentsTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Investments [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfInvestmentsTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfInvestmentsTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfInvestmentsTable" xlink:to="lab_us-gaap_ScheduleOfInvestmentsTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FaceamountofnotesdistributedfromRiskRetentionVehicle_3f1a344f-f974-4daf-a603-788d38dd2104_terseLabel_en-US" xlink:label="lab_efc_FaceamountofnotesdistributedfromRiskRetentionVehicle" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Face amount of notes distributed from Risk Retention Vehicle</link:label>
    <link:label id="lab_efc_FaceamountofnotesdistributedfromRiskRetentionVehicle_label_en-US" xlink:label="lab_efc_FaceamountofnotesdistributedfromRiskRetentionVehicle" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Face amount of notes distributed from Risk Retention Vehicle</link:label>
    <link:label id="lab_efc_FaceamountofnotesdistributedfromRiskRetentionVehicle_documentation_en-US" xlink:label="lab_efc_FaceamountofnotesdistributedfromRiskRetentionVehicle" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Face amount of notes distributed from Risk Retention Vehicle to the Company.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FaceamountofnotesdistributedfromRiskRetentionVehicle" xlink:href="efc-20201231.xsd#efc_FaceamountofnotesdistributedfromRiskRetentionVehicle"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FaceamountofnotesdistributedfromRiskRetentionVehicle" xlink:to="lab_efc_FaceamountofnotesdistributedfromRiskRetentionVehicle" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FinancialDerivativeAverageMonthlyNotional1_61de9c68-acaa-474e-a2c6-fc238c210a04_terseLabel_en-US" xlink:label="lab_efc_FinancialDerivativeAverageMonthlyNotional1" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Average Monthly Notional</link:label>
    <link:label id="lab_efc_FinancialDerivativeAverageMonthlyNotional1_label_en-US" xlink:label="lab_efc_FinancialDerivativeAverageMonthlyNotional1" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Derivative, Average Monthly Notional1</link:label>
    <link:label id="lab_efc_FinancialDerivativeAverageMonthlyNotional1_documentation_en-US" xlink:label="lab_efc_FinancialDerivativeAverageMonthlyNotional1" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Derivative, Average Monthly Notional1</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FinancialDerivativeAverageMonthlyNotional1" xlink:href="efc-20201231.xsd#efc_FinancialDerivativeAverageMonthlyNotional1"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FinancialDerivativeAverageMonthlyNotional1" xlink:to="lab_efc_FinancialDerivativeAverageMonthlyNotional1" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfQuarterlyFinancialInformationTableTextBlock_d696bd9e-7a03-4246-8556-485494865ebb_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfQuarterlyFinancialInformationTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Quarterly Financial Information</link:label>
    <link:label id="lab_us-gaap_ScheduleOfQuarterlyFinancialInformationTableTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfQuarterlyFinancialInformationTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Quarterly Financial Information [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfQuarterlyFinancialInformationTableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfQuarterlyFinancialInformationTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfQuarterlyFinancialInformationTableTextBlock" xlink:to="lab_us-gaap_ScheduleOfQuarterlyFinancialInformationTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_VestDecember122019Member_8323a5ce-7efb-4a7c-a18f-3ed8f0b824d8_terseLabel_en-US" xlink:label="lab_efc_VestDecember122019Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 12, 2019 [Member]</link:label>
    <link:label id="lab_efc_VestDecember122019Member_label_en-US" xlink:label="lab_efc_VestDecember122019Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 12, 2019 [Member]</link:label>
    <link:label id="lab_efc_VestDecember122019Member_documentation_en-US" xlink:label="lab_efc_VestDecember122019Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 12, 2019 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_VestDecember122019Member" xlink:href="efc-20201231.xsd#efc_VestDecember122019Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_VestDecember122019Member" xlink:to="lab_efc_VestDecember122019Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentOwnedBalanceShares_9b827af2-70a4-40e3-a0f0-23e538b316cb_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentOwnedBalanceShares" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Owned, Balance, Shares</link:label>
    <link:label id="lab_us-gaap_InvestmentOwnedBalanceShares_label_en-US" xlink:label="lab_us-gaap_InvestmentOwnedBalanceShares" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Owned, Balance, Shares</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentOwnedBalanceShares" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentOwnedBalanceShares"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentOwnedBalanceShares" xlink:to="lab_us-gaap_InvestmentOwnedBalanceShares" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProceedsFromMinorityShareholders_5dc5c6d2-6166-4216-a2e1-dbe7526525e2_terseLabel_en-US" xlink:label="lab_us-gaap_ProceedsFromMinorityShareholders" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Contributions from non-controlling interests</link:label>
    <link:label id="lab_us-gaap_ProceedsFromMinorityShareholders_label_en-US" xlink:label="lab_us-gaap_ProceedsFromMinorityShareholders" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from Noncontrolling Interests</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProceedsFromMinorityShareholders" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromMinorityShareholders"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProceedsFromMinorityShareholders" xlink:to="lab_us-gaap_ProceedsFromMinorityShareholders" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_VestDecember132021Member_a96d3e0f-215c-42b4-8098-005877e39eff_terseLabel_en-US" xlink:label="lab_efc_VestDecember132021Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 13, 2021 [Member]</link:label>
    <link:label id="lab_efc_VestDecember132021Member_label_en-US" xlink:label="lab_efc_VestDecember132021Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 13, 2021 [Member]</link:label>
    <link:label id="lab_efc_VestDecember132021Member_documentation_en-US" xlink:label="lab_efc_VestDecember132021Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 13, 2021 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_VestDecember132021Member" xlink:href="efc-20201231.xsd#efc_VestDecember132021Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_VestDecember132021Member" xlink:to="lab_efc_VestDecember132021Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MeasurementInputDiscountRateMember_23481858-ae4e-41d3-9d08-d61c173d1cf1_terseLabel_en-US" xlink:label="lab_us-gaap_MeasurementInputDiscountRateMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, Discount Rate</link:label>
    <link:label id="lab_us-gaap_MeasurementInputDiscountRateMember_label_en-US" xlink:label="lab_us-gaap_MeasurementInputDiscountRateMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, Discount Rate [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MeasurementInputDiscountRateMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MeasurementInputDiscountRateMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MeasurementInputDiscountRateMember" xlink:to="lab_us-gaap_MeasurementInputDiscountRateMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncomeApproachValuationTechniqueMember_1b0feaed-e7e0-4613-84ea-bb85c52c60d3_terseLabel_en-US" xlink:label="lab_us-gaap_IncomeApproachValuationTechniqueMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Valuation, Income Approach</link:label>
    <link:label id="lab_us-gaap_IncomeApproachValuationTechniqueMember_label_en-US" xlink:label="lab_us-gaap_IncomeApproachValuationTechniqueMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Valuation, Income Approach [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncomeApproachValuationTechniqueMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncomeApproachValuationTechniqueMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncomeApproachValuationTechniqueMember" xlink:to="lab_us-gaap_IncomeApproachValuationTechniqueMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year3Point50PercentMember_71a2977f-e2cf-4638-bf4c-ba502b3f4e89_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year3Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 3 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year3Point50PercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year3Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 3 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year3Point50PercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year3Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 3 Point 50 Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociation30Year3Point50PercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociation30Year3Point50PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociation30Year3Point50PercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociation30Year3Point50PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationTable_2a3a7235-2967-481b-9f51-973414ceeb03_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Liabilities Measured on Recurring Basis, Unobservable Input Reconciliation [Table]</link:label>
    <link:label id="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationTable_label_en-US" xlink:label="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Liabilities Measured on Recurring Basis, Unobservable Input Reconciliation [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationTable" xlink:to="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationTable" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_CoverAbstract_edda3454-a6eb-464c-a233-77a2db4cf191_terseLabel_en-US" xlink:label="lab_dei_CoverAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cover [Abstract]</link:label>
    <link:label id="lab_dei_CoverAbstract_label_en-US" xlink:label="lab_dei_CoverAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cover [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_CoverAbstract" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_CoverAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_CoverAbstract" xlink:to="lab_dei_CoverAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_DocumentType_f4def2f5-7a1b-4e90-9d81-1bdcee3def7c_terseLabel_en-US" xlink:label="lab_dei_DocumentType" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Document Type</link:label>
    <link:label id="lab_dei_DocumentType_label_en-US" xlink:label="lab_dei_DocumentType" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Document Type</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_DocumentType" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_DocumentType"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_DocumentType" xlink:to="lab_dei_DocumentType" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtInstrumentInterestRateEffectivePercentage_ef03ae05-4811-4883-bee1-7a9a8adc3f8b_terseLabel_en-US" xlink:label="lab_us-gaap_DebtInstrumentInterestRateEffectivePercentage" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Instrument, Interest Rate, Effective Rate</link:label>
    <link:label id="lab_us-gaap_DebtInstrumentInterestRateEffectivePercentage_label_en-US" xlink:label="lab_us-gaap_DebtInstrumentInterestRateEffectivePercentage" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Instrument, Interest Rate, Effective Percentage</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtInstrumentInterestRateEffectivePercentage" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtInstrumentInterestRateEffectivePercentage"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtInstrumentInterestRateEffectivePercentage" xlink:to="lab_us-gaap_DebtInstrumentInterestRateEffectivePercentage" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_TransferorAxis_bae511fa-f68d-44d4-8bc2-36f80a7366c1_terseLabel_en-US" xlink:label="lab_efc_TransferorAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transferor [Axis]</link:label>
    <link:label id="lab_efc_TransferorAxis_label_en-US" xlink:label="lab_efc_TransferorAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transferor [Axis]</link:label>
    <link:label id="lab_efc_TransferorAxis_documentation_en-US" xlink:label="lab_efc_TransferorAxis" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transferor [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_TransferorAxis" xlink:href="efc-20201231.xsd#efc_TransferorAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_TransferorAxis" xlink:to="lab_efc_TransferorAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTextBlock_bdad9c54-67e8-4136-a800-ef5689693c3b_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Measurement Using Significant Unobservable Inputs</link:label>
    <link:label id="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTextBlock_label_en-US" xlink:label="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Assets Measured on Recurring Basis, Unobservable Input Reconciliation [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTextBlock" xlink:to="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax_5a216670-4f60-4e74-8a06-d116ef4ad1b4_negatedLabel_en-US" xlink:label="lab_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Gross Unrealized Losses</link:label>
    <link:label id="lab_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax_fb6ecfb9-0286-406d-acd4-30e0fb187152_terseLabel_en-US" xlink:label="lab_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Accumulated Gross Unrealized Loss, before Tax</link:label>
    <link:label id="lab_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax_label_en-US" xlink:label="lab_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Accumulated Gross Unrealized Loss, before Tax</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax" xlink:to="lab_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedLossBeforeTax" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AdjustableRateResidentialMortgageMember_79b3d1b7-6d9d-4e02-b6f8-731e62d15916_terseLabel_en-US" xlink:label="lab_us-gaap_AdjustableRateResidentialMortgageMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Adjustable rate mortgages</link:label>
    <link:label id="lab_us-gaap_AdjustableRateResidentialMortgageMember_label_en-US" xlink:label="lab_us-gaap_AdjustableRateResidentialMortgageMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Adjustable Rate Residential Mortgage [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AdjustableRateResidentialMortgageMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AdjustableRateResidentialMortgageMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AdjustableRateResidentialMortgageMember" xlink:to="lab_us-gaap_AdjustableRateResidentialMortgageMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationThreePercentMember_d585a3a8-3c62-44c9-a424-72a454e6ffc1_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationThreePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Three Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationThreePercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationThreePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Three Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationThreePercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationThreePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Three Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationThreePercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationThreePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationThreePercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationThreePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FinancingReceivables90to119DaysPastDueMember_7223e17b-6b22-4360-9248-dd25aff4e852_verboseLabel_en-US" xlink:label="lab_efc_FinancingReceivables90to119DaysPastDueMember" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">90-119 Days or More Past Due [Member]</link:label>
    <link:label id="lab_efc_FinancingReceivables90to119DaysPastDueMember_label_en-US" xlink:label="lab_efc_FinancingReceivables90to119DaysPastDueMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivables, 90 to 119 Days Past Due [Member]</link:label>
    <link:label id="lab_efc_FinancingReceivables90to119DaysPastDueMember_documentation_en-US" xlink:label="lab_efc_FinancingReceivables90to119DaysPastDueMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing receivables that are 90 -119 days past due.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FinancingReceivables90to119DaysPastDueMember" xlink:href="efc-20201231.xsd#efc_FinancingReceivables90to119DaysPastDueMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FinancingReceivables90to119DaysPastDueMember" xlink:to="lab_efc_FinancingReceivables90to119DaysPastDueMember" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_ScenarioUnspecifiedDomain_a57ae9a5-83b3-464a-bde5-90899afa4698_terseLabel_en-US" xlink:label="lab_srt_ScenarioUnspecifiedDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Scenario [Domain]</link:label>
    <link:label id="lab_srt_ScenarioUnspecifiedDomain_label_en-US" xlink:label="lab_srt_ScenarioUnspecifiedDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Scenario [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_ScenarioUnspecifiedDomain" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_ScenarioUnspecifiedDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_ScenarioUnspecifiedDomain" xlink:to="lab_srt_ScenarioUnspecifiedDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DisclosureOfCompensationRelatedCostsSharebasedPaymentsAbstract_df7fa2c2-fc7d-400e-b350-a37a9b8a389c_terseLabel_en-US" xlink:label="lab_us-gaap_DisclosureOfCompensationRelatedCostsSharebasedPaymentsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Payment Arrangement [Abstract]</link:label>
    <link:label id="lab_us-gaap_DisclosureOfCompensationRelatedCostsSharebasedPaymentsAbstract_label_en-US" xlink:label="lab_us-gaap_DisclosureOfCompensationRelatedCostsSharebasedPaymentsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Payment Arrangement [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DisclosureOfCompensationRelatedCostsSharebasedPaymentsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DisclosureOfCompensationRelatedCostsSharebasedPaymentsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DisclosureOfCompensationRelatedCostsSharebasedPaymentsAbstract" xlink:to="lab_us-gaap_DisclosureOfCompensationRelatedCostsSharebasedPaymentsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureAbstract_cbcbb100-d166-441c-b99a-965ab933c308_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Instruments and Hedging Activities Disclosure [Abstract]</link:label>
    <link:label id="lab_us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureAbstract_label_en-US" xlink:label="lab_us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Instruments and Hedging Activities Disclosure [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureAbstract" xlink:to="lab_us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeAssetFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection_cc6aecb9-95fb-4ba3-a66a-995b8c157ba6_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeAssetFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Amount</link:label>
    <link:label id="lab_us-gaap_DerivativeAssetFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection_label_en-US" xlink:label="lab_us-gaap_DerivativeAssetFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Asset, Fair Value, Offset Against Collateral, Net of Not Subject to Master Netting Arrangement, Policy Election</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeAssetFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeAssetFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeAssetFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:to="lab_us-gaap_DerivativeAssetFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EnterpriseValueMember_11946782-593c-4e50-8f62-81d781d19fcd_terseLabel_en-US" xlink:label="lab_efc_EnterpriseValueMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Enterprise Value</link:label>
    <link:label id="lab_efc_EnterpriseValueMember_label_en-US" xlink:label="lab_efc_EnterpriseValueMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Enterprise Value [Member]</link:label>
    <link:label id="lab_efc_EnterpriseValueMember_documentation_en-US" xlink:label="lab_efc_EnterpriseValueMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Enterprise Value [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EnterpriseValueMember" xlink:href="efc-20201231.xsd#efc_EnterpriseValueMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EnterpriseValueMember" xlink:to="lab_efc_EnterpriseValueMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AvailableForSaleSecuritiesGrossRealizedGainLossNet_de4ed65e-17ac-4395-8837-66b15ae6a6be_terseLabel_en-US" xlink:label="lab_us-gaap_AvailableForSaleSecuritiesGrossRealizedGainLossNet" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Available-for-sale Securities, Gross Realized Gain (Loss)</link:label>
    <link:label id="lab_us-gaap_AvailableForSaleSecuritiesGrossRealizedGainLossNet_label_en-US" xlink:label="lab_us-gaap_AvailableForSaleSecuritiesGrossRealizedGainLossNet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Available-for-sale Securities, Gross Realized Gain (Loss)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AvailableForSaleSecuritiesGrossRealizedGainLossNet" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AvailableForSaleSecuritiesGrossRealizedGainLossNet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AvailableForSaleSecuritiesGrossRealizedGainLossNet" xlink:to="lab_us-gaap_AvailableForSaleSecuritiesGrossRealizedGainLossNet" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PrincipalPaymentOnSecuritizedDebt_b658de54-7e85-4ed0-bf3a-7390677fb432_negatedTerseLabel_en-US" xlink:label="lab_efc_PrincipalPaymentOnSecuritizedDebt" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal payment on securitized debt</link:label>
    <link:label id="lab_efc_PrincipalPaymentOnSecuritizedDebt_label_en-US" xlink:label="lab_efc_PrincipalPaymentOnSecuritizedDebt" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal payment on securitized debt</link:label>
    <link:label id="lab_efc_PrincipalPaymentOnSecuritizedDebt_documentation_en-US" xlink:label="lab_efc_PrincipalPaymentOnSecuritizedDebt" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The cash outflow for principal repayments on re-securitized assets accounted for as collateralized borrowings.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PrincipalPaymentOnSecuritizedDebt" xlink:href="efc-20201231.xsd#efc_PrincipalPaymentOnSecuritizedDebt"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PrincipalPaymentOnSecuritizedDebt" xlink:to="lab_efc_PrincipalPaymentOnSecuritizedDebt" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NumberOfWarrantsPurchasedUnderCommitmentAgreement_ff87fc40-bf3f-479a-b664-e998c4d9f965_terseLabel_en-US" xlink:label="lab_efc_NumberOfWarrantsPurchasedUnderCommitmentAgreement" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of warrants purchased under commitment agreement</link:label>
    <link:label id="lab_efc_NumberOfWarrantsPurchasedUnderCommitmentAgreement_label_en-US" xlink:label="lab_efc_NumberOfWarrantsPurchasedUnderCommitmentAgreement" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of warrants purchased under commitment agreement</link:label>
    <link:label id="lab_efc_NumberOfWarrantsPurchasedUnderCommitmentAgreement_documentation_en-US" xlink:label="lab_efc_NumberOfWarrantsPurchasedUnderCommitmentAgreement" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of warrants purchased under commitment agreement</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NumberOfWarrantsPurchasedUnderCommitmentAgreement" xlink:href="efc-20201231.xsd#efc_NumberOfWarrantsPurchasedUnderCommitmentAgreement"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NumberOfWarrantsPurchasedUnderCommitmentAgreement" xlink:to="lab_efc_NumberOfWarrantsPurchasedUnderCommitmentAgreement" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CommunicationsMember_ca5d3f55-b275-487f-a666-5df90f602e93_terseLabel_en-US" xlink:label="lab_efc_CommunicationsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Communications [Member]</link:label>
    <link:label id="lab_efc_CommunicationsMember_label_en-US" xlink:label="lab_efc_CommunicationsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Communications [Member]</link:label>
    <link:label id="lab_efc_CommunicationsMember_documentation_en-US" xlink:label="lab_efc_CommunicationsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Communications [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CommunicationsMember" xlink:href="efc-20201231.xsd#efc_CommunicationsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CommunicationsMember" xlink:to="lab_efc_CommunicationsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancialInstrumentAxis_fffd4769-3875-4d9f-9dcf-5e971314635c_terseLabel_en-US" xlink:label="lab_us-gaap_FinancialInstrumentAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instrument [Axis]</link:label>
    <link:label id="lab_us-gaap_FinancialInstrumentAxis_label_en-US" xlink:label="lab_us-gaap_FinancialInstrumentAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instrument [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancialInstrumentAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancialInstrumentAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancialInstrumentAxis" xlink:to="lab_us-gaap_FinancialInstrumentAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PreferredStockMember_bdb4b702-ee7d-42b0-9ca8-5d62acce4361_terseLabel_en-US" xlink:label="lab_us-gaap_PreferredStockMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock [Member]</link:label>
    <link:label id="lab_us-gaap_PreferredStockMember_label_en-US" xlink:label="lab_us-gaap_PreferredStockMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PreferredStockMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PreferredStockMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PreferredStockMember" xlink:to="lab_us-gaap_PreferredStockMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancingReceivablePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate_b6c50a5b-537e-4c40-93eb-0df56b026ce4_terseLabel_en-US" xlink:label="lab_us-gaap_FinancingReceivablePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable, Purchased with Credit Deterioration, Allowance for Credit Loss at Acquisition Date</link:label>
    <link:label id="lab_us-gaap_FinancingReceivablePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate_label_en-US" xlink:label="lab_us-gaap_FinancingReceivablePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable, Purchased with Credit Deterioration, Allowance for Credit Loss at Acquisition Date</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancingReceivablePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancingReceivablePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancingReceivablePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate" xlink:to="lab_us-gaap_FinancingReceivablePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansOnRealEstateLoanTypeAxis_b4664a3b-9490-4bb9-8c08-36d674984d30_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateLoanTypeAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Loan Type [Axis]</link:label>
    <link:label id="lab_srt_MortgageLoansOnRealEstateLoanTypeAxis_label_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateLoanTypeAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Loan Type [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansOnRealEstateLoanTypeAxis" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansOnRealEstateLoanTypeAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansOnRealEstateLoanTypeAxis" xlink:to="lab_srt_MortgageLoansOnRealEstateLoanTypeAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentCompanyGainLossOnInvestmentPerShare_03b6388e-5c88-40be-9334-9ebb7e505665_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentCompanyGainLossOnInvestmentPerShare" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Realized/Unrealized Gains (Losses)</link:label>
    <link:label id="lab_us-gaap_InvestmentCompanyGainLossOnInvestmentPerShare_label_en-US" xlink:label="lab_us-gaap_InvestmentCompanyGainLossOnInvestmentPerShare" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Gain (Loss) on Investment, Per Share</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentCompanyGainLossOnInvestmentPerShare" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentCompanyGainLossOnInvestmentPerShare"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentCompanyGainLossOnInvestmentPerShare" xlink:to="lab_us-gaap_InvestmentCompanyGainLossOnInvestmentPerShare" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OtherinvestmentrelatedexpensesServicingexpense_952a7ad9-e2be-41b5-863b-4605fdeef3c4_terseLabel_en-US" xlink:label="lab_efc_OtherinvestmentrelatedexpensesServicingexpense" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Servicing expense</link:label>
    <link:label id="lab_efc_OtherinvestmentrelatedexpensesServicingexpense_label_en-US" xlink:label="lab_efc_OtherinvestmentrelatedexpensesServicingexpense" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other investment related expenses - Servicing expense</link:label>
    <link:label id="lab_efc_OtherinvestmentrelatedexpensesServicingexpense_documentation_en-US" xlink:label="lab_efc_OtherinvestmentrelatedexpensesServicingexpense" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Servicing expense on loan portfolios. Included as Other Investment Related Expense.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OtherinvestmentrelatedexpensesServicingexpense" xlink:href="efc-20201231.xsd#efc_OtherinvestmentrelatedexpensesServicingexpense"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OtherinvestmentrelatedexpensesServicingexpense" xlink:to="lab_efc_OtherinvestmentrelatedexpensesServicingexpense" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PreferredStockSharesOutstanding_922b56ed-559e-44fc-841d-36ecce3863b2_terseLabel_en-US" xlink:label="lab_us-gaap_PreferredStockSharesOutstanding" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Shares Outstanding</link:label>
    <link:label id="lab_us-gaap_PreferredStockSharesOutstanding_label_en-US" xlink:label="lab_us-gaap_PreferredStockSharesOutstanding" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Shares Outstanding</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PreferredStockSharesOutstanding" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PreferredStockSharesOutstanding"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PreferredStockSharesOutstanding" xlink:to="lab_us-gaap_PreferredStockSharesOutstanding" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ResidentialMortgageMember_ac74c3ec-0ab3-46f5-bfaf-0da33938ca16_terseLabel_en-US" xlink:label="lab_us-gaap_ResidentialMortgageMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Residential mortgage loans</link:label>
    <link:label id="lab_us-gaap_ResidentialMortgageMember_label_en-US" xlink:label="lab_us-gaap_ResidentialMortgageMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Residential Mortgage [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ResidentialMortgageMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ResidentialMortgageMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ResidentialMortgageMember" xlink:to="lab_us-gaap_ResidentialMortgageMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EstimateweightedaveragelifeDomain_a6d174f1-9276-4294-bec4-f95dab4b46e3_terseLabel_en-US" xlink:label="lab_efc_EstimateweightedaveragelifeDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Estimate weighted average life [Domain]</link:label>
    <link:label id="lab_efc_EstimateweightedaveragelifeDomain_label_en-US" xlink:label="lab_efc_EstimateweightedaveragelifeDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Estimate weighted average life [Domain]</link:label>
    <link:label id="lab_efc_EstimateweightedaveragelifeDomain_documentation_en-US" xlink:label="lab_efc_EstimateweightedaveragelifeDomain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Domain] for Estimate weighted average life [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EstimateweightedaveragelifeDomain" xlink:href="efc-20201231.xsd#efc_EstimateweightedaveragelifeDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EstimateweightedaveragelifeDomain" xlink:to="lab_efc_EstimateweightedaveragelifeDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CommitmentsAndContingencies_fb509b89-6f72-4481-92f9-0370e56e283d_terseLabel_en-US" xlink:label="lab_us-gaap_CommitmentsAndContingencies" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commitments and contingencies</link:label>
    <link:label id="lab_us-gaap_CommitmentsAndContingencies_label_en-US" xlink:label="lab_us-gaap_CommitmentsAndContingencies" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commitments and Contingencies</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CommitmentsAndContingencies" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CommitmentsAndContingencies"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CommitmentsAndContingencies" xlink:to="lab_us-gaap_CommitmentsAndContingencies" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents_a9cc8c66-0233-4517-abdd-416a7792f483_periodStartLabel_en-US" xlink:label="lab_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash, Cash Equivalents, and Restricted Cash, Beginning of Period</link:label>
    <link:label id="lab_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents_4c885ca1-595f-445c-9445-5505b324a6d1_periodEndLabel_en-US" xlink:label="lab_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash, Cash Equivalents, and Restricted Cash, End of Period</link:label>
    <link:label id="lab_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents_label_en-US" xlink:label="lab_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash, Cash Equivalents, Restricted Cash and Restricted Cash Equivalents</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents" xlink:to="lab_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalents" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancialInstrumentsSoldNotYetPurchasedAtFairValue_06a369bb-8d99-4982-a15b-077fbb4d638d_terseLabel_en-US" xlink:label="lab_us-gaap_FinancialInstrumentsSoldNotYetPurchasedAtFairValue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities sold short, at fair value</link:label>
    <link:label id="lab_us-gaap_FinancialInstrumentsSoldNotYetPurchasedAtFairValue_9a80c61f-2372-4d9d-9948-c99a1ab17bf2_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_FinancialInstrumentsSoldNotYetPurchasedAtFairValue" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities sold short, at fair value</link:label>
    <link:label id="lab_us-gaap_FinancialInstrumentsSoldNotYetPurchasedAtFairValue_257fd37f-07bd-4fb2-ba9a-c822160f072a_negatedLabel_en-US" xlink:label="lab_us-gaap_FinancialInstrumentsSoldNotYetPurchasedAtFairValue" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments sold short, at fair value-</link:label>
    <link:label id="lab_us-gaap_FinancialInstrumentsSoldNotYetPurchasedAtFairValue_label_en-US" xlink:label="lab_us-gaap_FinancialInstrumentsSoldNotYetPurchasedAtFairValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instruments Sold, Not yet Purchased, at Fair Value</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancialInstrumentsSoldNotYetPurchasedAtFairValue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancialInstrumentsSoldNotYetPurchasedAtFairValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancialInstrumentsSoldNotYetPurchasedAtFairValue" xlink:to="lab_us-gaap_FinancialInstrumentsSoldNotYetPurchasedAtFairValue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_BankofAmericaSecuritiesMember_9a72e18f-ec7e-4eca-8fa4-97dc0e74c4f4_terseLabel_en-US" xlink:label="lab_efc_BankofAmericaSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of America Securities [Member]</link:label>
    <link:label id="lab_efc_BankofAmericaSecuritiesMember_label_en-US" xlink:label="lab_efc_BankofAmericaSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of America Securities [Member]</link:label>
    <link:label id="lab_efc_BankofAmericaSecuritiesMember_documentation_en-US" xlink:label="lab_efc_BankofAmericaSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of America Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_BankofAmericaSecuritiesMember" xlink:href="efc-20201231.xsd#efc_BankofAmericaSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_BankofAmericaSecuritiesMember" xlink:to="lab_efc_BankofAmericaSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtInstrumentLineItems_f3ca6320-6490-4211-964d-fdd50ba72292_terseLabel_en-US" xlink:label="lab_us-gaap_DebtInstrumentLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Instrument [Line Items]</link:label>
    <link:label id="lab_us-gaap_DebtInstrumentLineItems_label_en-US" xlink:label="lab_us-gaap_DebtInstrumentLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Instrument [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtInstrumentLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtInstrumentLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtInstrumentLineItems" xlink:to="lab_us-gaap_DebtInstrumentLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfVariableInterestEntitiesTable_7e164b08-1f04-4f4d-b78d-41d3e1fe0e2d_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfVariableInterestEntitiesTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Variable Interest Entities [Table]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfVariableInterestEntitiesTable_label_en-US" xlink:label="lab_us-gaap_ScheduleOfVariableInterestEntitiesTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Variable Interest Entities [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfVariableInterestEntitiesTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfVariableInterestEntitiesTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfVariableInterestEntitiesTable" xlink:to="lab_us-gaap_ScheduleOfVariableInterestEntitiesTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ResecuritizationOfGovernmentNationalMortgageAssociationMember_10a1867e-0614-4b4e-a205-66cf6bed334c_terseLabel_en-US" xlink:label="lab_efc_ResecuritizationOfGovernmentNationalMortgageAssociationMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Resecuritization of Government National Mortgage Association [Member]</link:label>
    <link:label id="lab_efc_ResecuritizationOfGovernmentNationalMortgageAssociationMember_label_en-US" xlink:label="lab_efc_ResecuritizationOfGovernmentNationalMortgageAssociationMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Resecuritization of Government National Mortgage Association [Member]</link:label>
    <link:label id="lab_efc_ResecuritizationOfGovernmentNationalMortgageAssociationMember_documentation_en-US" xlink:label="lab_efc_ResecuritizationOfGovernmentNationalMortgageAssociationMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Resecuritization of Government National Mortgage Association [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ResecuritizationOfGovernmentNationalMortgageAssociationMember" xlink:href="efc-20201231.xsd#efc_ResecuritizationOfGovernmentNationalMortgageAssociationMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ResecuritizationOfGovernmentNationalMortgageAssociationMember" xlink:to="lab_efc_ResecuritizationOfGovernmentNationalMortgageAssociationMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_WeightedAverageNumberOfShareOutstandingBasicAndDiluted_8ac67cac-4655-47f1-8489-cdb6eeb3c09b_terseLabel_en-US" xlink:label="lab_us-gaap_WeightedAverageNumberOfShareOutstandingBasicAndDiluted" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Weighted average shares outstanding (basic and diluted) (in shares)</link:label>
    <link:label id="lab_us-gaap_WeightedAverageNumberOfShareOutstandingBasicAndDiluted_label_en-US" xlink:label="lab_us-gaap_WeightedAverageNumberOfShareOutstandingBasicAndDiluted" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Weighted Average Number of Shares Outstanding, Basic and Diluted</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_WeightedAverageNumberOfShareOutstandingBasicAndDiluted" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_WeightedAverageNumberOfShareOutstandingBasicAndDiluted"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_WeightedAverageNumberOfShareOutstandingBasicAndDiluted" xlink:to="lab_us-gaap_WeightedAverageNumberOfShareOutstandingBasicAndDiluted" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CashAndCashEquivalentsPolicyTextBlock_6004d588-76fe-4b3d-ae83-3b9eb9212393_verboseLabel_en-US" xlink:label="lab_us-gaap_CashAndCashEquivalentsPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash and Cash Equivalents</link:label>
    <link:label id="lab_us-gaap_CashAndCashEquivalentsPolicyTextBlock_label_en-US" xlink:label="lab_us-gaap_CashAndCashEquivalentsPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash and Cash Equivalents, Policy [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CashAndCashEquivalentsPolicyTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CashAndCashEquivalentsPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CashAndCashEquivalentsPolicyTextBlock" xlink:to="lab_us-gaap_CashAndCashEquivalentsPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MinorityInterestInOperatingPartnerships_be32f9fa-05b6-4ca8-a0ca-626447140a7a_terseLabel_en-US" xlink:label="lab_us-gaap_MinorityInterestInOperatingPartnerships" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noncontrolling Interest in Operating Partnerships</link:label>
    <link:label id="lab_us-gaap_MinorityInterestInOperatingPartnerships_label_en-US" xlink:label="lab_us-gaap_MinorityInterestInOperatingPartnerships" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noncontrolling Interest in Operating Partnerships</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MinorityInterestInOperatingPartnerships" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MinorityInterestInOperatingPartnerships"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MinorityInterestInOperatingPartnerships" xlink:to="lab_us-gaap_MinorityInterestInOperatingPartnerships" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilityIssues_12cb2ed2-2375-4427-ab73-9dd6ff4665e7_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilityIssues" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities, Issuances</link:label>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilityIssues_label_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilityIssues" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement with Unobservable Inputs Reconciliation, Recurring Basis, Liability, Issuances</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilityIssues" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilityIssues"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilityIssues" xlink:to="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilityIssues" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansUnder5000000Member_d00e4274-f632-4fe7-9079-e35cf3d045b9_terseLabel_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansUnder5000000Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans under $5,000,000</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansUnder5000000Member_label_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansUnder5000000Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans under $5,000,000 [Member]</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansUnder5000000Member_documentation_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansUnder5000000Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans under $5,000,000</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansUnder5000000Member" xlink:href="efc-20201231.xsd#efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansUnder5000000Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansUnder5000000Member" xlink:to="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansUnder5000000Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LoansheldforinvestmentMember_e5e747e6-42c9-4a4b-a84e-82c725417a06_terseLabel_en-US" xlink:label="lab_efc_LoansheldforinvestmentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans held-for-investment [Member]</link:label>
    <link:label id="lab_efc_LoansheldforinvestmentMember_label_en-US" xlink:label="lab_efc_LoansheldforinvestmentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans held-for-investment [Member]</link:label>
    <link:label id="lab_efc_LoansheldforinvestmentMember_documentation_en-US" xlink:label="lab_efc_LoansheldforinvestmentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans held-for-investment [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LoansheldforinvestmentMember" xlink:href="efc-20201231.xsd#efc_LoansheldforinvestmentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LoansheldforinvestmentMember" xlink:to="lab_efc_LoansheldforinvestmentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MeasurementInputRecoveryAmount_1d6267bb-e343-42f3-b7f8-c9611dd23751_terseLabel_en-US" xlink:label="lab_efc_MeasurementInputRecoveryAmount" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input recovery amount</link:label>
    <link:label id="lab_efc_MeasurementInputRecoveryAmount_label_en-US" xlink:label="lab_efc_MeasurementInputRecoveryAmount" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input recovery amount</link:label>
    <link:label id="lab_efc_MeasurementInputRecoveryAmount_documentation_en-US" xlink:label="lab_efc_MeasurementInputRecoveryAmount" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input recovery amount</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MeasurementInputRecoveryAmount" xlink:href="efc-20201231.xsd#efc_MeasurementInputRecoveryAmount"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MeasurementInputRecoveryAmount" xlink:to="lab_efc_MeasurementInputRecoveryAmount" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearThreePointFivePercentMember_91491149-176a-4b98-b910-9d61f5ab1cac_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Fifteen Year) Three Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearThreePointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Fifteen Year) Three Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearThreePointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Fifteen Year) Three Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearThreePointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearThreePointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearThreePointFivePercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationPoolsFifteenYearThreePointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationByAssetClassDomain_95a96892-910d-4e24-afd3-2ff559c0fc20_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationByAssetClassDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Asset Class [Domain]</link:label>
    <link:label id="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationByAssetClassDomain_label_en-US" xlink:label="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationByAssetClassDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Asset Class [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationByAssetClassDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationByAssetClassDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationByAssetClassDomain" xlink:to="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationByAssetClassDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PrivateLabelSecuritiesMember_3b0687f4-1187-46c8-ab5b-fafc73345a31_terseLabel_en-US" xlink:label="lab_efc_PrivateLabelSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Private Label Securities [Member]</link:label>
    <link:label id="lab_efc_PrivateLabelSecuritiesMember_label_en-US" xlink:label="lab_efc_PrivateLabelSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Private Label Securities [Member]</link:label>
    <link:label id="lab_efc_PrivateLabelSecuritiesMember_documentation_en-US" xlink:label="lab_efc_PrivateLabelSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Private label securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PrivateLabelSecuritiesMember" xlink:href="efc-20201231.xsd#efc_PrivateLabelSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PrivateLabelSecuritiesMember" xlink:to="lab_efc_PrivateLabelSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NumberOfWarrantsAvailableForPurchaseUnderCommitmentAgreement_3d71c9ce-9a21-46db-b92c-6cd63c151ff0_terseLabel_en-US" xlink:label="lab_efc_NumberOfWarrantsAvailableForPurchaseUnderCommitmentAgreement" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of warrants available for purchase under commitment agreement</link:label>
    <link:label id="lab_efc_NumberOfWarrantsAvailableForPurchaseUnderCommitmentAgreement_label_en-US" xlink:label="lab_efc_NumberOfWarrantsAvailableForPurchaseUnderCommitmentAgreement" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of warrants available for purchase under commitment agreement</link:label>
    <link:label id="lab_efc_NumberOfWarrantsAvailableForPurchaseUnderCommitmentAgreement_documentation_en-US" xlink:label="lab_efc_NumberOfWarrantsAvailableForPurchaseUnderCommitmentAgreement" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of warrants available for purchase under commitment agreement</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NumberOfWarrantsAvailableForPurchaseUnderCommitmentAgreement" xlink:href="efc-20201231.xsd#efc_NumberOfWarrantsAvailableForPurchaseUnderCommitmentAgreement"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NumberOfWarrantsAvailableForPurchaseUnderCommitmentAgreement" xlink:to="lab_efc_NumberOfWarrantsAvailableForPurchaseUnderCommitmentAgreement" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ResidentialMortgageandCommercialMortgageBackedSecuritiesMember_c0361a17-bc34-4804-ab24-d08af8090ac1_terseLabel_en-US" xlink:label="lab_efc_ResidentialMortgageandCommercialMortgageBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Agency RMBS and CMBS</link:label>
    <link:label id="lab_efc_ResidentialMortgageandCommercialMortgageBackedSecuritiesMember_label_en-US" xlink:label="lab_efc_ResidentialMortgageandCommercialMortgageBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Residential Mortgage and Commercial Mortgage Backed Securities [Member]</link:label>
    <link:label id="lab_efc_ResidentialMortgageandCommercialMortgageBackedSecuritiesMember_documentation_en-US" xlink:label="lab_efc_ResidentialMortgageandCommercialMortgageBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Residential Mortgage and Commercial Mortgage Backed Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ResidentialMortgageandCommercialMortgageBackedSecuritiesMember" xlink:href="efc-20201231.xsd#efc_ResidentialMortgageandCommercialMortgageBackedSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ResidentialMortgageandCommercialMortgageBackedSecuritiesMember" xlink:to="lab_efc_ResidentialMortgageandCommercialMortgageBackedSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_November132018Member_96ee36fd-8a3b-45c3-a737-c27d3b66dcc2_terseLabel_en-US" xlink:label="lab_efc_November132018Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">November 2018 [Member]</link:label>
    <link:label id="lab_efc_November132018Member_label_en-US" xlink:label="lab_efc_November132018Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">November 13, 2018 [Member]</link:label>
    <link:label id="lab_efc_November132018Member_documentation_en-US" xlink:label="lab_efc_November132018Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">November 13, 2018 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_November132018Member" xlink:href="efc-20201231.xsd#efc_November132018Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_November132018Member" xlink:to="lab_efc_November132018Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OtherCommitmentsLineItems_a2cc9019-24f5-4661-9972-5d856a88122a_terseLabel_en-US" xlink:label="lab_us-gaap_OtherCommitmentsLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Commitments [Line Items]</link:label>
    <link:label id="lab_us-gaap_OtherCommitmentsLineItems_label_en-US" xlink:label="lab_us-gaap_OtherCommitmentsLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Commitments [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OtherCommitmentsLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OtherCommitmentsLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OtherCommitmentsLineItems" xlink:to="lab_us-gaap_OtherCommitmentsLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CLOIIISecuritizationMember_99725ee2-5091-4a6d-8634-0540244bdcf9_terseLabel_en-US" xlink:label="lab_efc_CLOIIISecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLO III Securitization [Member]</link:label>
    <link:label id="lab_efc_CLOIIISecuritizationMember_label_en-US" xlink:label="lab_efc_CLOIIISecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLO III Securitization [Member]</link:label>
    <link:label id="lab_efc_CLOIIISecuritizationMember_documentation_en-US" xlink:label="lab_efc_CLOIIISecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLO II Securitization (CLO transaction priced June 2018 closed July 2018)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CLOIIISecuritizationMember" xlink:href="efc-20201231.xsd#efc_CLOIIISecuritizationMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CLOIIISecuritizationMember" xlink:to="lab_efc_CLOIIISecuritizationMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardAwardVestingPeriod1_54db0986-6315-4b94-a69f-43af4cb37810_verboseLabel_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardAwardVestingPeriod1" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vesting period</link:label>
    <link:label id="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardAwardVestingPeriod1_label_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardAwardVestingPeriod1" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Compensation Arrangement by Share-based Payment Award, Award Vesting Period</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardAwardVestingPeriod1" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardAwardVestingPeriod1"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardAwardVestingPeriod1" xlink:to="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardAwardVestingPeriod1" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Sharebasedlongtermincentiveplanunitredemptionanddistribution_07cfed76-2966-41bd-92f3-1c0c215b9aca_terseLabel_en-US" xlink:label="lab_efc_Sharebasedlongtermincentiveplanunitredemptionanddistribution" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based long term incentive plan unit redemption and distribution</link:label>
    <link:label id="lab_efc_Sharebasedlongtermincentiveplanunitredemptionanddistribution_label_en-US" xlink:label="lab_efc_Sharebasedlongtermincentiveplanunitredemptionanddistribution" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based long term incentive plan unit redemption and distribution</link:label>
    <link:label id="lab_efc_Sharebasedlongtermincentiveplanunitredemptionanddistribution_documentation_en-US" xlink:label="lab_efc_Sharebasedlongtermincentiveplanunitredemptionanddistribution" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based long term incentive plan unit redemption and distribution</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Sharebasedlongtermincentiveplanunitredemptionanddistribution" xlink:href="efc-20201231.xsd#efc_Sharebasedlongtermincentiveplanunitredemptionanddistribution"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Sharebasedlongtermincentiveplanunitredemptionanddistribution" xlink:to="lab_efc_Sharebasedlongtermincentiveplanunitredemptionanddistribution" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationBorrowings_fa6779ee-3f9b-4341-b21d-56b3f866f3b2_terseLabel_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationBorrowings" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Borrowings</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationBorrowings_label_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationBorrowings" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Borrowings</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationBorrowings_documentation_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationBorrowings" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The amount of borrowings (including secured borrowings, notes, and lines of credit) reported by an equity method investment of the entity.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationBorrowings" xlink:href="efc-20201231.xsd#efc_EquityMethodInvestmentSummarizedFinancialInformationBorrowings"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationBorrowings" xlink:to="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationBorrowings" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ForeignExchangeFutureMember_a48e2436-6fac-4918-bfd8-3a456afc1142_terseLabel_en-US" xlink:label="lab_us-gaap_ForeignExchangeFutureMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Foreign Exchange Future [Member]</link:label>
    <link:label id="lab_us-gaap_ForeignExchangeFutureMember_label_en-US" xlink:label="lab_us-gaap_ForeignExchangeFutureMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Foreign Exchange Future [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ForeignExchangeFutureMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ForeignExchangeFutureMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ForeignExchangeFutureMember" xlink:to="lab_us-gaap_ForeignExchangeFutureMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MeasurementInputRecoveryAmountMember_a66d12e1-9dce-4d0b-b75c-16665cea81fc_terseLabel_en-US" xlink:label="lab_efc_MeasurementInputRecoveryAmountMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input recovery amount</link:label>
    <link:label id="lab_efc_MeasurementInputRecoveryAmountMember_label_en-US" xlink:label="lab_efc_MeasurementInputRecoveryAmountMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input recovery amount [Member]</link:label>
    <link:label id="lab_efc_MeasurementInputRecoveryAmountMember_documentation_en-US" xlink:label="lab_efc_MeasurementInputRecoveryAmountMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input recovery amount</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MeasurementInputRecoveryAmountMember" xlink:href="efc-20201231.xsd#efc_MeasurementInputRecoveryAmountMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MeasurementInputRecoveryAmountMember" xlink:to="lab_efc_MeasurementInputRecoveryAmountMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Otheroperatingexpenses_dd17916e-6d60-4c26-acd3-60bd75d02228_terseLabel_en-US" xlink:label="lab_efc_Otheroperatingexpenses" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other operating expenses</link:label>
    <link:label id="lab_efc_Otheroperatingexpenses_label_en-US" xlink:label="lab_efc_Otheroperatingexpenses" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other operating expenses</link:label>
    <link:label id="lab_efc_Otheroperatingexpenses_documentation_en-US" xlink:label="lab_efc_Otheroperatingexpenses" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Sum of all operating expenses incurred as disclosed on the Statement of Operations excluding the following expenses: Base management fee, Incentive fee, Interest Expense, and Other investment related expenses.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Otheroperatingexpenses" xlink:href="efc-20201231.xsd#efc_Otheroperatingexpenses"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Otheroperatingexpenses" xlink:to="lab_efc_Otheroperatingexpenses" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SchedulesOfConcentrationOfRiskByRiskFactorTextBlock_7ed52d95-9feb-4d19-94d9-a50f2bdbe7f0_verboseLabel_en-US" xlink:label="lab_us-gaap_SchedulesOfConcentrationOfRiskByRiskFactorTextBlock" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedules of Concentration of Risk, by Risk Factor [Table Text Block]</link:label>
    <link:label id="lab_us-gaap_SchedulesOfConcentrationOfRiskByRiskFactorTextBlock_c14e2bd1-097a-4832-a1f8-cba569dff8e3_terseLabel_en-US" xlink:label="lab_us-gaap_SchedulesOfConcentrationOfRiskByRiskFactorTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedules of Exposure to Counterparty Risk</link:label>
    <link:label id="lab_us-gaap_SchedulesOfConcentrationOfRiskByRiskFactorTextBlock_label_en-US" xlink:label="lab_us-gaap_SchedulesOfConcentrationOfRiskByRiskFactorTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedules of Concentration of Risk, by Risk Factor [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SchedulesOfConcentrationOfRiskByRiskFactorTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SchedulesOfConcentrationOfRiskByRiskFactorTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SchedulesOfConcentrationOfRiskByRiskFactorTextBlock" xlink:to="lab_us-gaap_SchedulesOfConcentrationOfRiskByRiskFactorTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LongInvestmentsMember_fb5f53be-22a9-41dd-b63c-cce488248340_terseLabel_en-US" xlink:label="lab_efc_LongInvestmentsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investments [Member]</link:label>
    <link:label id="lab_efc_LongInvestmentsMember_label_en-US" xlink:label="lab_efc_LongInvestmentsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investments [Member]</link:label>
    <link:label id="lab_efc_LongInvestmentsMember_documentation_en-US" xlink:label="lab_efc_LongInvestmentsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The Company's long investments held as an asset in the statement of financial position.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LongInvestmentsMember" xlink:href="efc-20201231.xsd#efc_LongInvestmentsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LongInvestmentsMember" xlink:to="lab_efc_LongInvestmentsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeLiabilityNotOffsetPolicyElectionDeduction_03ad23d1-6ee0-42f1-a90d-41398c9487c7_negatedLabel_en-US" xlink:label="lab_us-gaap_DerivativeLiabilityNotOffsetPolicyElectionDeduction" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instruments Available for Offset</link:label>
    <link:label id="lab_us-gaap_DerivativeLiabilityNotOffsetPolicyElectionDeduction_label_en-US" xlink:label="lab_us-gaap_DerivativeLiabilityNotOffsetPolicyElectionDeduction" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Liability, Not Offset, Policy Election Deduction</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeLiabilityNotOffsetPolicyElectionDeduction" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeLiabilityNotOffsetPolicyElectionDeduction"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeLiabilityNotOffsetPolicyElectionDeduction" xlink:to="lab_us-gaap_DerivativeLiabilityNotOffsetPolicyElectionDeduction" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NoninterestIncomeOtherOperatingIncome_35c2a4ce-9ab3-471d-95a8-5623bc4060da_terseLabel_en-US" xlink:label="lab_us-gaap_NoninterestIncomeOtherOperatingIncome" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other, net</link:label>
    <link:label id="lab_us-gaap_NoninterestIncomeOtherOperatingIncome_label_en-US" xlink:label="lab_us-gaap_NoninterestIncomeOtherOperatingIncome" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noninterest Income, Other Operating Income</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NoninterestIncomeOtherOperatingIncome" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NoninterestIncomeOtherOperatingIncome"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NoninterestIncomeOtherOperatingIncome" xlink:to="lab_us-gaap_NoninterestIncomeOtherOperatingIncome" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember_f868ba18-cab7-4ecb-8865-9248c7f82de4_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Three Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Three Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Three Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationSixPercentMember_1c4b17fd-83f9-4b3d-8bfb-e8c33fee69c3_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationSixPercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Six Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationSixPercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationSixPercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Six Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationSixPercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationSixPercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Six Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationSixPercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationSixPercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationSixPercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationSixPercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PreferredStockValueOutstanding_8c1c5673-ed28-4447-9c32-e9e89e244c54_verboseLabel_en-US" xlink:label="lab_us-gaap_PreferredStockValueOutstanding" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred stock, par value $0.001 per share, 100,000,000 shares authorized;6.750% Series A Fixed-to-Floating Rate Cumulative Redeemable; 4,600,000 shares issued and outstanding ($115,000 liquidation preference)</link:label>
    <link:label id="lab_us-gaap_PreferredStockValueOutstanding_fb7b5a30-0c4f-4d9e-a06a-b8581c2b079d_terseLabel_en-US" xlink:label="lab_us-gaap_PreferredStockValueOutstanding" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Value, Outstanding</link:label>
    <link:label id="lab_us-gaap_PreferredStockValueOutstanding_label_en-US" xlink:label="lab_us-gaap_PreferredStockValueOutstanding" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Value, Outstanding</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PreferredStockValueOutstanding" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PreferredStockValueOutstanding"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PreferredStockValueOutstanding" xlink:to="lab_us-gaap_PreferredStockValueOutstanding" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InvestmentRelatedReceivablesPolicyTextBlock_41d336b5-c1a9-474f-aaa7-ef1b03ce097a_terseLabel_en-US" xlink:label="lab_efc_InvestmentRelatedReceivablesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Related Receivables [Policy Text Block]</link:label>
    <link:label id="lab_efc_InvestmentRelatedReceivablesPolicyTextBlock_label_en-US" xlink:label="lab_efc_InvestmentRelatedReceivablesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Related Receivables [Policy Text Block]</link:label>
    <link:label id="lab_efc_InvestmentRelatedReceivablesPolicyTextBlock_documentation_en-US" xlink:label="lab_efc_InvestmentRelatedReceivablesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Related Receivables [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InvestmentRelatedReceivablesPolicyTextBlock" xlink:href="efc-20201231.xsd#efc_InvestmentRelatedReceivablesPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InvestmentRelatedReceivablesPolicyTextBlock" xlink:to="lab_efc_InvestmentRelatedReceivablesPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LongInvestmentUnratedRatingMember_5981e171-d50e-490d-89e8-d27d8c888138_terseLabel_en-US" xlink:label="lab_efc_LongInvestmentUnratedRatingMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Unrated Rating [Member]</link:label>
    <link:label id="lab_efc_LongInvestmentUnratedRatingMember_label_en-US" xlink:label="lab_efc_LongInvestmentUnratedRatingMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Unrated Rating [Member]</link:label>
    <link:label id="lab_efc_LongInvestmentUnratedRatingMember_documentation_en-US" xlink:label="lab_efc_LongInvestmentUnratedRatingMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">No credit rating available from Moody's, Standards and Poor's, or Fitch.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LongInvestmentUnratedRatingMember" xlink:href="efc-20201231.xsd#efc_LongInvestmentUnratedRatingMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LongInvestmentUnratedRatingMember" xlink:to="lab_efc_LongInvestmentUnratedRatingMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_UnsecuredloansMember_e1cdd5a8-41dd-461a-90a7-058f77567bef_terseLabel_en-US" xlink:label="lab_efc_UnsecuredloansMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unsecured loans</link:label>
    <link:label id="lab_efc_UnsecuredloansMember_label_en-US" xlink:label="lab_efc_UnsecuredloansMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unsecured loans [Member]</link:label>
    <link:label id="lab_efc_UnsecuredloansMember_documentation_en-US" xlink:label="lab_efc_UnsecuredloansMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unsecured loans held as investments.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_UnsecuredloansMember" xlink:href="efc-20201231.xsd#efc_UnsecuredloansMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_UnsecuredloansMember" xlink:to="lab_efc_UnsecuredloansMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RepurchaseAgreementsamountatriskthreshold_4bfbcb7c-ff50-4d22-9727-d9d16ea07a19_terseLabel_en-US" xlink:label="lab_efc_RepurchaseAgreementsamountatriskthreshold" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase Agreements amount at risk threshold</link:label>
    <link:label id="lab_efc_RepurchaseAgreementsamountatriskthreshold_label_en-US" xlink:label="lab_efc_RepurchaseAgreementsamountatriskthreshold" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase Agreements amount at risk threshold</link:label>
    <link:label id="lab_efc_RepurchaseAgreementsamountatriskthreshold_documentation_en-US" xlink:label="lab_efc_RepurchaseAgreementsamountatriskthreshold" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration risk threshold whereby disclosure of amount at risk by counterparty is required if amount at risk as a percentage of total equity is above such threshold.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RepurchaseAgreementsamountatriskthreshold" xlink:href="efc-20201231.xsd#efc_RepurchaseAgreementsamountatriskthreshold"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RepurchaseAgreementsamountatriskthreshold" xlink:to="lab_efc_RepurchaseAgreementsamountatriskthreshold" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SharesIssuedInConnectionWithIncentiveFeePaymentNonCash_9985d7c8-06e2-4844-a953-f8096e90e92c_verboseLabel_en-US" xlink:label="lab_efc_SharesIssuedInConnectionWithIncentiveFeePaymentNonCash" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Shares Issued In Connection With Incentive Fee Payment</link:label>
    <link:label id="lab_efc_SharesIssuedInConnectionWithIncentiveFeePaymentNonCash_8b1369dc-cc01-416e-b3ae-cf2fc88cf399_terseLabel_en-US" xlink:label="lab_efc_SharesIssuedInConnectionWithIncentiveFeePaymentNonCash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Shares Issued In Connection With Incentive Fee Payment Non-Cash</link:label>
    <link:label id="lab_efc_SharesIssuedInConnectionWithIncentiveFeePaymentNonCash_label_en-US" xlink:label="lab_efc_SharesIssuedInConnectionWithIncentiveFeePaymentNonCash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Shares Issued In Connection With Incentive Fee Payment Non-Cash</link:label>
    <link:label id="lab_efc_SharesIssuedInConnectionWithIncentiveFeePaymentNonCash_documentation_en-US" xlink:label="lab_efc_SharesIssuedInConnectionWithIncentiveFeePaymentNonCash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Value of shares issued to the Manager as partial payment for incentive fees.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SharesIssuedInConnectionWithIncentiveFeePaymentNonCash" xlink:href="efc-20201231.xsd#efc_SharesIssuedInConnectionWithIncentiveFeePaymentNonCash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SharesIssuedInConnectionWithIncentiveFeePaymentNonCash" xlink:to="lab_efc_SharesIssuedInConnectionWithIncentiveFeePaymentNonCash" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTable_876a0010-9d82-4afa-a5e4-8b35a472d599_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Share-based Compensation Arrangements by Share-based Payment Award [Table]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTable_label_en-US" xlink:label="lab_us-gaap_ScheduleOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Share-based Compensation Arrangements by Share-based Payment Award [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTable" xlink:to="lab_us-gaap_ScheduleOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EarningsPerShareDilutedAbstract_18c89810-ced4-4fa9-abf1-2de12b09c1ad_terseLabel_en-US" xlink:label="lab_us-gaap_EarningsPerShareDilutedAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Diluted earnings per share of common stock:</link:label>
    <link:label id="lab_us-gaap_EarningsPerShareDilutedAbstract_label_en-US" xlink:label="lab_us-gaap_EarningsPerShareDilutedAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share, Diluted [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EarningsPerShareDilutedAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EarningsPerShareDilutedAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EarningsPerShareDilutedAbstract" xlink:to="lab_us-gaap_EarningsPerShareDilutedAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OperatingIncomeLoss_798455c7-f53f-41d1-8518-541c39d30ce3_terseLabel_en-US" xlink:label="lab_us-gaap_OperatingIncomeLoss" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net investment income</link:label>
    <link:label id="lab_us-gaap_OperatingIncomeLoss_label_en-US" xlink:label="lab_us-gaap_OperatingIncomeLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Income (Loss)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OperatingIncomeLoss" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OperatingIncomeLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OperatingIncomeLoss" xlink:to="lab_us-gaap_OperatingIncomeLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ProceedsFromPrincipalPaymentsOfInvestments_8d95d035-d9a1-4a7f-9361-54c94c341305_terseLabel_en-US" xlink:label="lab_efc_ProceedsFromPrincipalPaymentsOfInvestments" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from principal payments of investments</link:label>
    <link:label id="lab_efc_ProceedsFromPrincipalPaymentsOfInvestments_label_en-US" xlink:label="lab_efc_ProceedsFromPrincipalPaymentsOfInvestments" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds From Principal Payments of Investments</link:label>
    <link:label id="lab_efc_ProceedsFromPrincipalPaymentsOfInvestments_documentation_en-US" xlink:label="lab_efc_ProceedsFromPrincipalPaymentsOfInvestments" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The cash inflow associated with the maturity and collection of all investments such as debt during the period.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ProceedsFromPrincipalPaymentsOfInvestments" xlink:href="efc-20201231.xsd#efc_ProceedsFromPrincipalPaymentsOfInvestments"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ProceedsFromPrincipalPaymentsOfInvestments" xlink:to="lab_efc_ProceedsFromPrincipalPaymentsOfInvestments" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2049Member_3cec6e27-58a1-43ff-b4e3-b72c833aa850_terseLabel_en-US" xlink:label="lab_efc_A2049Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2049 [Member]</link:label>
    <link:label id="lab_efc_A2049Member_label_en-US" xlink:label="lab_efc_A2049Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2049 [Member]</link:label>
    <link:label id="lab_efc_A2049Member_documentation_en-US" xlink:label="lab_efc_A2049Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2049 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2049Member" xlink:href="efc-20201231.xsd#efc_A2049Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2049Member" xlink:to="lab_efc_A2049Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonperformingsecuritizedresidentialmortgageloansMember_48ad0846-d27c-4e36-bb66-b691d6e3aa25_terseLabel_en-US" xlink:label="lab_efc_NonperformingsecuritizedresidentialmortgageloansMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitized loans [Member]</link:label>
    <link:label id="lab_efc_NonperformingsecuritizedresidentialmortgageloansMember_label_en-US" xlink:label="lab_efc_NonperformingsecuritizedresidentialmortgageloansMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-performing securitized residential mortgage loans [Member]</link:label>
    <link:label id="lab_efc_NonperformingsecuritizedresidentialmortgageloansMember_documentation_en-US" xlink:label="lab_efc_NonperformingsecuritizedresidentialmortgageloansMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-performing securitized residential mortgage loans [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonperformingsecuritizedresidentialmortgageloansMember" xlink:href="efc-20201231.xsd#efc_NonperformingsecuritizedresidentialmortgageloansMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonperformingsecuritizedresidentialmortgageloansMember" xlink:to="lab_efc_NonperformingsecuritizedresidentialmortgageloansMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancialInstrumentPerformanceStatusAxis_8f30c486-4001-4e6e-b4f2-6e671c17f997_terseLabel_en-US" xlink:label="lab_us-gaap_FinancialInstrumentPerformanceStatusAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instrument Performance Status [Axis]</link:label>
    <link:label id="lab_us-gaap_FinancialInstrumentPerformanceStatusAxis_label_en-US" xlink:label="lab_us-gaap_FinancialInstrumentPerformanceStatusAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instrument Performance Status [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancialInstrumentPerformanceStatusAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancialInstrumentPerformanceStatusAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancialInstrumentPerformanceStatusAxis" xlink:to="lab_us-gaap_FinancialInstrumentPerformanceStatusAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ReceivablesFromSecuritiesSoldFinancialDerivativesInterestandPrincipal_acbad972-6d9a-4b93-846c-ff4ff1f6aaaa_terseLabel_en-US" xlink:label="lab_efc_ReceivablesFromSecuritiesSoldFinancialDerivativesInterestandPrincipal" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment related receivables</link:label>
    <link:label id="lab_efc_ReceivablesFromSecuritiesSoldFinancialDerivativesInterestandPrincipal_label_en-US" xlink:label="lab_efc_ReceivablesFromSecuritiesSoldFinancialDerivativesInterestandPrincipal" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Receivables From Securities Sold, Financial Derivatives, Interest and Principal</link:label>
    <link:label id="lab_efc_ReceivablesFromSecuritiesSoldFinancialDerivativesInterestandPrincipal_documentation_en-US" xlink:label="lab_efc_ReceivablesFromSecuritiesSoldFinancialDerivativesInterestandPrincipal" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Receivables From Securities Sold, Financial Derivatives, Interest and Principal</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ReceivablesFromSecuritiesSoldFinancialDerivativesInterestandPrincipal" xlink:href="efc-20201231.xsd#efc_ReceivablesFromSecuritiesSoldFinancialDerivativesInterestandPrincipal"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ReceivablesFromSecuritiesSoldFinancialDerivativesInterestandPrincipal" xlink:to="lab_efc_ReceivablesFromSecuritiesSoldFinancialDerivativesInterestandPrincipal" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween15000000And19999999Member_1f5c1500-c097-4f8c-b94c-df5e02d14443_terseLabel_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween15000000And19999999Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $15,000,000 and $19,999,999</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween15000000And19999999Member_label_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween15000000And19999999Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $15,000,000 and $19,999,999 [Member]</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween15000000And19999999Member_documentation_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween15000000And19999999Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $15,000,000 and $19,999,999</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween15000000And19999999Member" xlink:href="efc-20201231.xsd#efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween15000000And19999999Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween15000000And19999999Member" xlink:to="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween15000000And19999999Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_IncreaseDecreaseInRepurchaseAgreements_c6db255b-f19e-44d6-a13c-032b251c5836_negatedTerseLabel_en-US" xlink:label="lab_efc_IncreaseDecreaseInRepurchaseAgreements" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase agreements</link:label>
    <link:label id="lab_efc_IncreaseDecreaseInRepurchaseAgreements_label_en-US" xlink:label="lab_efc_IncreaseDecreaseInRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase decrease in repurchase agreements</link:label>
    <link:label id="lab_efc_IncreaseDecreaseInRepurchaseAgreements_documentation_en-US" xlink:label="lab_efc_IncreaseDecreaseInRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The increase (decrease) during the reporting period in the total amount under agreements to purchase and resell securities.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_IncreaseDecreaseInRepurchaseAgreements" xlink:href="efc-20201231.xsd#efc_IncreaseDecreaseInRepurchaseAgreements"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_IncreaseDecreaseInRepurchaseAgreements" xlink:to="lab_efc_IncreaseDecreaseInRepurchaseAgreements" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncreaseDecreaseInOperatingLiabilitiesAbstract_c1cac208-d9c9-4511-b7ac-b274762ce7fd_terseLabel_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInOperatingLiabilitiesAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (decrease) in liabilities:</link:label>
    <link:label id="lab_us-gaap_IncreaseDecreaseInOperatingLiabilitiesAbstract_label_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInOperatingLiabilitiesAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (Decrease) in Operating Liabilities [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncreaseDecreaseInOperatingLiabilitiesAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncreaseDecreaseInOperatingLiabilitiesAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncreaseDecreaseInOperatingLiabilitiesAbstract" xlink:to="lab_us-gaap_IncreaseDecreaseInOperatingLiabilitiesAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityChangeInNetUnrealizedGainLoss_6f2436e0-e173-4375-b929-bc1e385ed01c_terseLabel_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityChangeInNetUnrealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities, Change In Net Unrealized Gain/(Loss)</link:label>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityChangeInNetUnrealizedGainLoss_label_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityChangeInNetUnrealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement With Unobservable Inputs Reconciliations, Recurring Basis, Liability, Change In Net Unrealized Gain/(Loss)</link:label>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityChangeInNetUnrealizedGainLoss_documentation_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityChangeInNetUnrealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement With Unobservable Inputs Reconciliations, Recurring Basis, Liability, Change In Net Unrealized Gain/(Loss)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityChangeInNetUnrealizedGainLoss" xlink:href="efc-20201231.xsd#efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityChangeInNetUnrealizedGainLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityChangeInNetUnrealizedGainLoss" xlink:to="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityChangeInNetUnrealizedGainLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_June2020Member_fc6b9c6b-9269-45c1-8d7e-3effbb3cedc2_terseLabel_en-US" xlink:label="lab_efc_June2020Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">June 2020 [Member]</link:label>
    <link:label id="lab_efc_June2020Member_label_en-US" xlink:label="lab_efc_June2020Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">June 2020 [Member]</link:label>
    <link:label id="lab_efc_June2020Member_documentation_en-US" xlink:label="lab_efc_June2020Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">June 2020 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_June2020Member" xlink:href="efc-20201231.xsd#efc_June2020Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_June2020Member" xlink:to="lab_efc_June2020Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NonperformingFinancingReceivableMember_19d668cf-d470-4cd7-a306-e7bb5027931c_terseLabel_en-US" xlink:label="lab_us-gaap_NonperformingFinancingReceivableMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-performing</link:label>
    <link:label id="lab_us-gaap_NonperformingFinancingReceivableMember_label_en-US" xlink:label="lab_us-gaap_NonperformingFinancingReceivableMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Nonperforming Financial Instruments [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NonperformingFinancingReceivableMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NonperformingFinancingReceivableMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NonperformingFinancingReceivableMember" xlink:to="lab_us-gaap_NonperformingFinancingReceivableMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ConsolidatedCondensedScheduleOfInvestmentsCashEquivalentsAbstract_label_en-US" xlink:label="lab_efc_ConsolidatedCondensedScheduleOfInvestmentsCashEquivalentsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidated condensed schedule of investments (cash equivalents) [Abstract]</link:label>
    <link:label id="lab_efc_ConsolidatedCondensedScheduleOfInvestmentsCashEquivalentsAbstract_documentation_en-US" xlink:label="lab_efc_ConsolidatedCondensedScheduleOfInvestmentsCashEquivalentsAbstract" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidated condensed schedule of investments (cash equivalents)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ConsolidatedCondensedScheduleOfInvestmentsCashEquivalentsAbstract" xlink:href="efc-20201231.xsd#efc_ConsolidatedCondensedScheduleOfInvestmentsCashEquivalentsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ConsolidatedCondensedScheduleOfInvestmentsCashEquivalentsAbstract" xlink:to="lab_efc_ConsolidatedCondensedScheduleOfInvestmentsCashEquivalentsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueMeasurementFrequencyDomain_7d3023f6-5b69-4b3f-82e7-bc8d85753552_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueMeasurementFrequencyDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Frequency [Domain]</link:label>
    <link:label id="lab_us-gaap_FairValueMeasurementFrequencyDomain_label_en-US" xlink:label="lab_us-gaap_FairValueMeasurementFrequencyDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Frequency [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueMeasurementFrequencyDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueMeasurementFrequencyDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueMeasurementFrequencyDomain" xlink:to="lab_us-gaap_FairValueMeasurementFrequencyDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValuesDerivativesBalanceSheetLocationByDerivativeContractTypeByHedgingDesignationTable_7420fa7f-925d-46bb-99ab-ebccf6a3b12d_terseLabel_en-US" xlink:label="lab_us-gaap_FairValuesDerivativesBalanceSheetLocationByDerivativeContractTypeByHedgingDesignationTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Values Derivatives, Balance Sheet Location, by Derivative Contract Type [Table]</link:label>
    <link:label id="lab_us-gaap_FairValuesDerivativesBalanceSheetLocationByDerivativeContractTypeByHedgingDesignationTable_label_en-US" xlink:label="lab_us-gaap_FairValuesDerivativesBalanceSheetLocationByDerivativeContractTypeByHedgingDesignationTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Values Derivatives, Balance Sheet Location, by Derivative Contract Type [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValuesDerivativesBalanceSheetLocationByDerivativeContractTypeByHedgingDesignationTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValuesDerivativesBalanceSheetLocationByDerivativeContractTypeByHedgingDesignationTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValuesDerivativesBalanceSheetLocationByDerivativeContractTypeByHedgingDesignationTable" xlink:to="lab_us-gaap_FairValuesDerivativesBalanceSheetLocationByDerivativeContractTypeByHedgingDesignationTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FinancingReceivablesEqualtoGreaterthan120DaysPastDueMember_fc96ace4-7864-43e9-b395-3ba7560d941c_terseLabel_en-US" xlink:label="lab_efc_FinancingReceivablesEqualtoGreaterthan120DaysPastDueMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivables, Equal to Greater than 120 Days Past Due [Member]</link:label>
    <link:label id="lab_efc_FinancingReceivablesEqualtoGreaterthan120DaysPastDueMember_label_en-US" xlink:label="lab_efc_FinancingReceivablesEqualtoGreaterthan120DaysPastDueMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivables, Equal to Greater than 120 Days Past Due [Member]</link:label>
    <link:label id="lab_efc_FinancingReceivablesEqualtoGreaterthan120DaysPastDueMember_documentation_en-US" xlink:label="lab_efc_FinancingReceivablesEqualtoGreaterthan120DaysPastDueMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivables, Equal to Greater than 120 Days Past Due [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FinancingReceivablesEqualtoGreaterthan120DaysPastDueMember" xlink:href="efc-20201231.xsd#efc_FinancingReceivablesEqualtoGreaterthan120DaysPastDueMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FinancingReceivablesEqualtoGreaterthan120DaysPastDueMember" xlink:to="lab_efc_FinancingReceivablesEqualtoGreaterthan120DaysPastDueMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EarningsPerShareBasic_413af211-791c-440d-a44b-c783cba7ea68_totalLabel_en-US" xlink:label="lab_us-gaap_EarningsPerShareBasic" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Basic earnings per common share (in usd per share)</link:label>
    <link:label id="lab_us-gaap_EarningsPerShareBasic_label_en-US" xlink:label="lab_us-gaap_EarningsPerShareBasic" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share, Basic</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EarningsPerShareBasic" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EarningsPerShareBasic"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EarningsPerShareBasic" xlink:to="lab_us-gaap_EarningsPerShareBasic" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RelatedPartyCLOsecuritizationMember_2798e618-cd22-480f-84db-4a2b8b469508_terseLabel_en-US" xlink:label="lab_efc_RelatedPartyCLOsecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party CLO securitization [Member]</link:label>
    <link:label id="lab_efc_RelatedPartyCLOsecuritizationMember_label_en-US" xlink:label="lab_efc_RelatedPartyCLOsecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party CLO securitization [Member]</link:label>
    <link:label id="lab_efc_RelatedPartyCLOsecuritizationMember_documentation_en-US" xlink:label="lab_efc_RelatedPartyCLOsecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party CLO securitization [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RelatedPartyCLOsecuritizationMember" xlink:href="efc-20201231.xsd#efc_RelatedPartyCLOsecuritizationMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RelatedPartyCLOsecuritizationMember" xlink:to="lab_efc_RelatedPartyCLOsecuritizationMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DerivativeactivityvolumeTableTextBlock_e5b5f670-3601-4e02-ae42-861bf91511fd_terseLabel_en-US" xlink:label="lab_efc_DerivativeactivityvolumeTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative activity, volume</link:label>
    <link:label id="lab_efc_DerivativeactivityvolumeTableTextBlock_label_en-US" xlink:label="lab_efc_DerivativeactivityvolumeTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative activity, volume [Table Text Block]</link:label>
    <link:label id="lab_efc_DerivativeactivityvolumeTableTextBlock_documentation_en-US" xlink:label="lab_efc_DerivativeactivityvolumeTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disclosure of the volume of derivative activity for the period. Calculated using average monthly notional.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DerivativeactivityvolumeTableTextBlock" xlink:href="efc-20201231.xsd#efc_DerivativeactivityvolumeTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DerivativeactivityvolumeTableTextBlock" xlink:to="lab_efc_DerivativeactivityvolumeTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CreditConcentrationRiskMember_15970ce6-4e6d-44e0-9062-bb5049821218_terseLabel_en-US" xlink:label="lab_us-gaap_CreditConcentrationRiskMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Counterparty Risk [Member]</link:label>
    <link:label id="lab_us-gaap_CreditConcentrationRiskMember_label_en-US" xlink:label="lab_us-gaap_CreditConcentrationRiskMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Concentration Risk [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CreditConcentrationRiskMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CreditConcentrationRiskMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CreditConcentrationRiskMember" xlink:to="lab_us-gaap_CreditConcentrationRiskMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2027Member_14f0234d-6976-4bcb-9bdf-618e52198b36_terseLabel_en-US" xlink:label="lab_efc_A2027Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2027 [Member]</link:label>
    <link:label id="lab_efc_A2027Member_label_en-US" xlink:label="lab_efc_A2027Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2027 [Member]</link:label>
    <link:label id="lab_efc_A2027Member_documentation_en-US" xlink:label="lab_efc_A2027Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2027 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2027Member" xlink:href="efc-20201231.xsd#efc_A2027Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2027Member" xlink:to="lab_efc_A2027Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SignificantAccountingPoliciesLineItems_d1e8b1a3-d2b4-4c02-bdfa-8aedb09a4294_terseLabel_en-US" xlink:label="lab_efc_SignificantAccountingPoliciesLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Significant Accounting Policies [Line Items]</link:label>
    <link:label id="lab_efc_SignificantAccountingPoliciesLineItems_label_en-US" xlink:label="lab_efc_SignificantAccountingPoliciesLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Significant Accounting Policies [Line Items]</link:label>
    <link:label id="lab_efc_SignificantAccountingPoliciesLineItems_documentation_en-US" xlink:label="lab_efc_SignificantAccountingPoliciesLineItems" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Significant Accounting Policies [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SignificantAccountingPoliciesLineItems" xlink:href="efc-20201231.xsd#efc_SignificantAccountingPoliciesLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SignificantAccountingPoliciesLineItems" xlink:to="lab_efc_SignificantAccountingPoliciesLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ToBeAnnouncedSecuritiesTextBlock_e9ee7f29-7e68-4a47-bb73-25546b305c9f_terseLabel_en-US" xlink:label="lab_efc_ToBeAnnouncedSecuritiesTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">To Be Announced RMBS</link:label>
    <link:label id="lab_efc_ToBeAnnouncedSecuritiesTextBlock_label_en-US" xlink:label="lab_efc_ToBeAnnouncedSecuritiesTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">To Be Announced Securities [Text Block]</link:label>
    <link:label id="lab_efc_ToBeAnnouncedSecuritiesTextBlock_documentation_en-US" xlink:label="lab_efc_ToBeAnnouncedSecuritiesTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The complete disclosure for forward settling transactions in the To Be Announced ("TBA") MBS market.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ToBeAnnouncedSecuritiesTextBlock" xlink:href="efc-20201231.xsd#efc_ToBeAnnouncedSecuritiesTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ToBeAnnouncedSecuritiesTextBlock" xlink:to="lab_efc_ToBeAnnouncedSecuritiesTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PaymentsToAcquireLoansReceivable_682ad440-b21c-4558-83f6-18f657b01031_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_PaymentsToAcquireLoansReceivable" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchase of loans</link:label>
    <link:label id="lab_us-gaap_PaymentsToAcquireLoansReceivable_label_en-US" xlink:label="lab_us-gaap_PaymentsToAcquireLoansReceivable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments to Acquire Loans Receivable</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PaymentsToAcquireLoansReceivable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsToAcquireLoansReceivable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PaymentsToAcquireLoansReceivable" xlink:to="lab_us-gaap_PaymentsToAcquireLoansReceivable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OfferingCostsPlacementFeesPolicyTextBlock_effa74e3-8b9d-42ff-b470-fd88d4725784_terseLabel_en-US" xlink:label="lab_efc_OfferingCostsPlacementFeesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Offering Costs/Underwriters' Discount</link:label>
    <link:label id="lab_efc_OfferingCostsPlacementFeesPolicyTextBlock_label_en-US" xlink:label="lab_efc_OfferingCostsPlacementFeesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Offering Costs Placement Fees [Policy Text Block]</link:label>
    <link:label id="lab_efc_OfferingCostsPlacementFeesPolicyTextBlock_documentation_en-US" xlink:label="lab_efc_OfferingCostsPlacementFeesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disclosure of accounting policy for treatment of offering costs and placement fees.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OfferingCostsPlacementFeesPolicyTextBlock" xlink:href="efc-20201231.xsd#efc_OfferingCostsPlacementFeesPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OfferingCostsPlacementFeesPolicyTextBlock" xlink:to="lab_efc_OfferingCostsPlacementFeesPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ConsolidatedEntitiesMember_71fc15b3-6f2e-40db-8908-e94968cffcf7_terseLabel_en-US" xlink:label="lab_us-gaap_ConsolidatedEntitiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidated Entities [Member]</link:label>
    <link:label id="lab_us-gaap_ConsolidatedEntitiesMember_label_en-US" xlink:label="lab_us-gaap_ConsolidatedEntitiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidated Entities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ConsolidatedEntitiesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ConsolidatedEntitiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ConsolidatedEntitiesMember" xlink:to="lab_us-gaap_ConsolidatedEntitiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SubsequentEventTypeAxis_8170be9d-cd11-43db-973f-e059e66b2a44_terseLabel_en-US" xlink:label="lab_us-gaap_SubsequentEventTypeAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsequent Event Type [Axis]</link:label>
    <link:label id="lab_us-gaap_SubsequentEventTypeAxis_label_en-US" xlink:label="lab_us-gaap_SubsequentEventTypeAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsequent Event Type [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SubsequentEventTypeAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SubsequentEventTypeAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SubsequentEventTypeAxis" xlink:to="lab_us-gaap_SubsequentEventTypeAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ConcentrationRiskBenchmarkDomain_bc8ff5c2-7eb4-4483-affe-46bdc3eadd99_terseLabel_en-US" xlink:label="lab_us-gaap_ConcentrationRiskBenchmarkDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk Benchmark [Domain]</link:label>
    <link:label id="lab_us-gaap_ConcentrationRiskBenchmarkDomain_label_en-US" xlink:label="lab_us-gaap_ConcentrationRiskBenchmarkDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk Benchmark [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ConcentrationRiskBenchmarkDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ConcentrationRiskBenchmarkDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ConcentrationRiskBenchmarkDomain" xlink:to="lab_us-gaap_ConcentrationRiskBenchmarkDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisChangeInUnrealizedGainLoss_bdc421b6-fa2b-4580-9cc4-c0399a2fdd71_verboseLabel_en-US" xlink:label="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisChangeInUnrealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Change in unrealized gain (loss), assets</link:label>
    <link:label id="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisChangeInUnrealizedGainLoss_09e7b3c6-d316-4397-9ebd-6b60ca3a0a6b_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisChangeInUnrealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Change in unrealized gains (losses)</link:label>
    <link:label id="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisChangeInUnrealizedGainLoss_label_en-US" xlink:label="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisChangeInUnrealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Asset, Recurring Basis, Still Held, Unrealized Gain (Loss)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueAssetsMeasuredOnRecurringBasisChangeInUnrealizedGainLoss" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueAssetsMeasuredOnRecurringBasisChangeInUnrealizedGainLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueAssetsMeasuredOnRecurringBasisChangeInUnrealizedGainLoss" xlink:to="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisChangeInUnrealizedGainLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationCalculationRollForward_a81f5608-a643-468f-bedc-8a0f376d1ee7_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationCalculationRollForward" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Assets Measured on Recurring Basis, Unobservable Input Reconciliation, Calculation [Roll Forward]</link:label>
    <link:label id="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationCalculationRollForward_label_en-US" xlink:label="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationCalculationRollForward" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Assets Measured on Recurring Basis, Unobservable Input Reconciliation, Calculation [Roll Forward]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationCalculationRollForward" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationCalculationRollForward"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationCalculationRollForward" xlink:to="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationCalculationRollForward" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_TBASecuritiesPurchaseContractsMember_79c6a5a8-953f-48a7-bafa-c9888176cb43_terseLabel_en-US" xlink:label="lab_efc_TBASecuritiesPurchaseContractsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">TBA Securities, Purchase Contracts [Member]</link:label>
    <link:label id="lab_efc_TBASecuritiesPurchaseContractsMember_label_en-US" xlink:label="lab_efc_TBASecuritiesPurchaseContractsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">TBA Securities, Purchase Contracts [Member]</link:label>
    <link:label id="lab_efc_TBASecuritiesPurchaseContractsMember_documentation_en-US" xlink:label="lab_efc_TBASecuritiesPurchaseContractsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Forward contract for the purchase (long position) of Agency RMBS in the To-be-announced RMBS market.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_TBASecuritiesPurchaseContractsMember" xlink:href="efc-20201231.xsd#efc_TBASecuritiesPurchaseContractsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_TBASecuritiesPurchaseContractsMember" xlink:to="lab_efc_TBASecuritiesPurchaseContractsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StatementOfStockholdersEquityAbstract_550d9ca0-ae8c-42b3-9d9c-c1a412666a35_terseLabel_en-US" xlink:label="lab_us-gaap_StatementOfStockholdersEquityAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Statement of Stockholders' Equity [Abstract]</link:label>
    <link:label id="lab_us-gaap_StatementOfStockholdersEquityAbstract_label_en-US" xlink:label="lab_us-gaap_StatementOfStockholdersEquityAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Statement of Stockholders' Equity [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StatementOfStockholdersEquityAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StatementOfStockholdersEquityAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StatementOfStockholdersEquityAbstract" xlink:to="lab_us-gaap_StatementOfStockholdersEquityAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncentiveFeeExpense_3172785a-ee36-437d-afe7-245ead866806_terseLabel_en-US" xlink:label="lab_us-gaap_IncentiveFeeExpense" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Incentive Fee Expense</link:label>
    <link:label id="lab_us-gaap_IncentiveFeeExpense_label_en-US" xlink:label="lab_us-gaap_IncentiveFeeExpense" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Incentive Fee Expense</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncentiveFeeExpense" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncentiveFeeExpense"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncentiveFeeExpense" xlink:to="lab_us-gaap_IncentiveFeeExpense" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleOfCommonSharesOutstandingTableTextBlock_e6a4129b-8f2f-4093-b846-3a75371dde54_terseLabel_en-US" xlink:label="lab_efc_ScheduleOfCommonSharesOutstandingTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Summary of Common Shares Outstanding</link:label>
    <link:label id="lab_efc_ScheduleOfCommonSharesOutstandingTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleOfCommonSharesOutstandingTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Common Shares Outstanding [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleOfCommonSharesOutstandingTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleOfCommonSharesOutstandingTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Common Shares Outstanding [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleOfCommonSharesOutstandingTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleOfCommonSharesOutstandingTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleOfCommonSharesOutstandingTableTextBlock" xlink:to="lab_efc_ScheduleOfCommonSharesOutstandingTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchase_3b55e999-d26a-46a1-8659-3e88cf325a3f_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchase" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reverse repurchase agreements</link:label>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchase_be35c326-a804-417b-82a9-c163ec789a79_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchase" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reverse repurchase agreements</link:label>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchase_label_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchase" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Sold under Agreements to Repurchase</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchase" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesSoldUnderAgreementsToRepurchase"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchase" xlink:to="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchase" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtInstrumentMaturityDate_1bc3a0df-758d-46c9-8330-70e23b61b5ac_terseLabel_en-US" xlink:label="lab_us-gaap_DebtInstrumentMaturityDate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Instrument, Maturity Date</link:label>
    <link:label id="lab_us-gaap_DebtInstrumentMaturityDate_label_en-US" xlink:label="lab_us-gaap_DebtInstrumentMaturityDate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Instrument, Maturity Date</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtInstrumentMaturityDate" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtInstrumentMaturityDate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtInstrumentMaturityDate" xlink:to="lab_us-gaap_DebtInstrumentMaturityDate" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationFivePointFivePercentMember_e24cab8b-6a9a-4e1c-acc3-a9f0ea71932f_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationFivePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Five Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationFivePointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationFivePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Five Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationFivePointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationFivePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Five Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationFivePointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationFivePointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationFivePointFivePercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationFivePointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ConsolidatedsecuritizationtrustsMember_02fbdecc-1706-4885-9a3d-aca3f056c1c6_terseLabel_en-US" xlink:label="lab_efc_ConsolidatedsecuritizationtrustsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidated securitization trusts [Member]</link:label>
    <link:label id="lab_efc_ConsolidatedsecuritizationtrustsMember_label_en-US" xlink:label="lab_efc_ConsolidatedsecuritizationtrustsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidated securitization trusts [Member]</link:label>
    <link:label id="lab_efc_ConsolidatedsecuritizationtrustsMember_documentation_en-US" xlink:label="lab_efc_ConsolidatedsecuritizationtrustsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidated securitization trusts [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ConsolidatedsecuritizationtrustsMember" xlink:href="efc-20201231.xsd#efc_ConsolidatedsecuritizationtrustsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ConsolidatedsecuritizationtrustsMember" xlink:to="lab_efc_ConsolidatedsecuritizationtrustsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Realizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet_0487678b-eb07-43c4-9a87-28126fdee1d9_terseLabel_en-US" xlink:label="lab_efc_Realizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized gains (losses) on securities and loans, financial derivatives, and real estate owned, net</link:label>
    <link:label id="lab_efc_Realizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet_label_en-US" xlink:label="lab_efc_Realizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized gains (losses) on securities and loans, financial derivatives, and real estate owned, net</link:label>
    <link:label id="lab_efc_Realizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet_documentation_en-US" xlink:label="lab_efc_Realizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized gains (losses) on securities and loans, financial derivatives, and real estate owned, net</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Realizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet" xlink:href="efc-20201231.xsd#efc_Realizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Realizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet" xlink:to="lab_efc_Realizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments_272644fa-9892-4b28-9aa9-39c65f8c1740_totalLabel_en-US" xlink:label="lab_us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Income (Loss) before Income Tax Expense (Benefit) and Earnings (Losses) from Investments in Unconsolidated Entities</link:label>
    <link:label id="lab_us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments_5161c489-d2e0-4e14-ae07-56b423fa169e_terseLabel_en-US" xlink:label="lab_us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Income (Loss) from Continuing Operations before Equity Method Investments, Income Taxes, Noncontrolling Interest</link:label>
    <link:label id="lab_us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments_label_en-US" xlink:label="lab_us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Income (Loss) from Continuing Operations before Equity Method Investments, Income Taxes, Noncontrolling Interest</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:to="lab_us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePercentMember_9bec7931-32cc-445e-a95a-01a3190feecc_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Fifteen Year) Three Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Fifteen Year) Three Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Fifteen Year) Three Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MorganStanleyInstitutionalLiquidityFundGovernmentPortfolioMember_1569fbde-4481-4548-bdfa-0cd9e37b0bca_terseLabel_en-US" xlink:label="lab_efc_MorganStanleyInstitutionalLiquidityFundGovernmentPortfolioMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Morgan Stanley Institutional Liquidity Fund - Government Portfolio [Member]</link:label>
    <link:label id="lab_efc_MorganStanleyInstitutionalLiquidityFundGovernmentPortfolioMember_label_en-US" xlink:label="lab_efc_MorganStanleyInstitutionalLiquidityFundGovernmentPortfolioMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Morgan Stanley Institutional Liquidity Fund - Government Portfolio [Member]</link:label>
    <link:label id="lab_efc_MorganStanleyInstitutionalLiquidityFundGovernmentPortfolioMember_documentation_en-US" xlink:label="lab_efc_MorganStanleyInstitutionalLiquidityFundGovernmentPortfolioMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Morgan Stanley Institutional Liquidity Fund - Government Portfolio [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MorganStanleyInstitutionalLiquidityFundGovernmentPortfolioMember" xlink:href="efc-20201231.xsd#efc_MorganStanleyInstitutionalLiquidityFundGovernmentPortfolioMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MorganStanleyInstitutionalLiquidityFundGovernmentPortfolioMember" xlink:to="lab_efc_MorganStanleyInstitutionalLiquidityFundGovernmentPortfolioMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_TypeOfRecipientAxis_798c35e3-c928-4de1-8f51-29d29f4f6593_terseLabel_en-US" xlink:label="lab_efc_TypeOfRecipientAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Type of Recipient [Axis]</link:label>
    <link:label id="lab_efc_TypeOfRecipientAxis_label_en-US" xlink:label="lab_efc_TypeOfRecipientAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Type of Recipient [Axis]</link:label>
    <link:label id="lab_efc_TypeOfRecipientAxis_documentation_en-US" xlink:label="lab_efc_TypeOfRecipientAxis" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disclosure by type of recipient related to share-based payment arrangements (LTIP awards).</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_TypeOfRecipientAxis" xlink:href="efc-20201231.xsd#efc_TypeOfRecipientAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_TypeOfRecipientAxis" xlink:to="lab_efc_TypeOfRecipientAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InterestExpense_aeaa3d70-3fc5-4f23-93fe-79c68df43a23_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_InterestExpense" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest expense</link:label>
    <link:label id="lab_us-gaap_InterestExpense_675d05ff-4a8d-4fee-b020-bdea738f503d_terseLabel_en-US" xlink:label="lab_us-gaap_InterestExpense" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Expense</link:label>
    <link:label id="lab_us-gaap_InterestExpense_label_en-US" xlink:label="lab_us-gaap_InterestExpense" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Expense</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InterestExpense" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InterestExpense"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InterestExpense" xlink:to="lab_us-gaap_InterestExpense" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RatioOfInterestandotherinvestmentrelatedExpensesToAverageEquity_a6167abd-2882-41d3-855d-4b3be8175585_negatedTerseLabel_en-US" xlink:label="lab_efc_RatioOfInterestandotherinvestmentrelatedExpensesToAverageEquity" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ratio Of Interest and other investment related Expenses To Average Equity</link:label>
    <link:label id="lab_efc_RatioOfInterestandotherinvestmentrelatedExpensesToAverageEquity_label_en-US" xlink:label="lab_efc_RatioOfInterestandotherinvestmentrelatedExpensesToAverageEquity" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ratio Of Interest and other investment related Expenses To Average Equity</link:label>
    <link:label id="lab_efc_RatioOfInterestandotherinvestmentrelatedExpensesToAverageEquity_documentation_en-US" xlink:label="lab_efc_RatioOfInterestandotherinvestmentrelatedExpensesToAverageEquity" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The ratio of interest and other investment related expenses to average total equity (including non-controlling interest).</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RatioOfInterestandotherinvestmentrelatedExpensesToAverageEquity" xlink:href="efc-20201231.xsd#efc_RatioOfInterestandotherinvestmentrelatedExpensesToAverageEquity"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RatioOfInterestandotherinvestmentrelatedExpensesToAverageEquity" xlink:to="lab_efc_RatioOfInterestandotherinvestmentrelatedExpensesToAverageEquity" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RestrictedCashAndCashEquivalents_1092d7ab-d5c8-4665-b5e7-24875435e7ed_terseLabel_en-US" xlink:label="lab_us-gaap_RestrictedCashAndCashEquivalents" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Restricted cash</link:label>
    <link:label id="lab_us-gaap_RestrictedCashAndCashEquivalents_label_en-US" xlink:label="lab_us-gaap_RestrictedCashAndCashEquivalents" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Restricted Cash and Cash Equivalents</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RestrictedCashAndCashEquivalents" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RestrictedCashAndCashEquivalents"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RestrictedCashAndCashEquivalents" xlink:to="lab_us-gaap_RestrictedCashAndCashEquivalents" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StatementClassOfStockAxis_71abd4d6-93e6-4e85-b24a-1a5edc94fdf6_terseLabel_en-US" xlink:label="lab_us-gaap_StatementClassOfStockAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Class of Stock [Axis]</link:label>
    <link:label id="lab_us-gaap_StatementClassOfStockAxis_label_en-US" xlink:label="lab_us-gaap_StatementClassOfStockAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Class of Stock [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StatementClassOfStockAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StatementClassOfStockAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StatementClassOfStockAxis" xlink:to="lab_us-gaap_StatementClassOfStockAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_TerminationFeeDescription_e96b4365-4706-4a74-ba79-2ded2cd7816b_terseLabel_en-US" xlink:label="lab_efc_TerminationFeeDescription" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Termination fee, description</link:label>
    <link:label id="lab_efc_TerminationFeeDescription_label_en-US" xlink:label="lab_efc_TerminationFeeDescription" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Termination Fee, Description</link:label>
    <link:label id="lab_efc_TerminationFeeDescription_documentation_en-US" xlink:label="lab_efc_TerminationFeeDescription" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Description of payments to managing member for termination or non-renewal of management agreement. Includes events of termination and basis for calculation.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_TerminationFeeDescription" xlink:href="efc-20201231.xsd#efc_TerminationFeeDescription"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_TerminationFeeDescription" xlink:to="lab_efc_TerminationFeeDescription" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsDisclosureTextBlock_64cdf38e-a3ed-42e9-b804-61ce2de5357b_terseLabel_en-US" xlink:label="lab_us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Organization and Investment Objective</link:label>
    <link:label id="lab_us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsDisclosureTextBlock_label_en-US" xlink:label="lab_us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Organization, Consolidation and Presentation of Financial Statements Disclosure [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsDisclosureTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsDisclosureTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsDisclosureTextBlock" xlink:to="lab_us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsDisclosureTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OffsettingOfAssetsAndLiabilitiesTextBlock_f1325e29-bfc5-478c-a376-11a9fa17a6ba_terseLabel_en-US" xlink:label="lab_efc_OffsettingOfAssetsAndLiabilitiesTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Offsetting of Assets and Liabilities</link:label>
    <link:label id="lab_efc_OffsettingOfAssetsAndLiabilitiesTextBlock_label_en-US" xlink:label="lab_efc_OffsettingOfAssetsAndLiabilitiesTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Offsetting of Assets and Liabilities [Text Block]</link:label>
    <link:label id="lab_efc_OffsettingOfAssetsAndLiabilitiesTextBlock_documentation_en-US" xlink:label="lab_efc_OffsettingOfAssetsAndLiabilitiesTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The entire disclosure for derivatives, repurchase agreements, and reverse repurchase agreements that are subject to offsetting. [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OffsettingOfAssetsAndLiabilitiesTextBlock" xlink:href="efc-20201231.xsd#efc_OffsettingOfAssetsAndLiabilitiesTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OffsettingOfAssetsAndLiabilitiesTextBlock" xlink:to="lab_efc_OffsettingOfAssetsAndLiabilitiesTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2028Member_e1a340be-4e8b-4a68-a646-5c08efae6534_terseLabel_en-US" xlink:label="lab_efc_A2028Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2028 [Member]</link:label>
    <link:label id="lab_efc_A2028Member_label_en-US" xlink:label="lab_efc_A2028Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2028 [Member]</link:label>
    <link:label id="lab_efc_A2028Member_documentation_en-US" xlink:label="lab_efc_A2028Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2028 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2028Member" xlink:href="efc-20201231.xsd#efc_A2028Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2028Member" xlink:to="lab_efc_A2028Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancialLiabilitiesFairValueDisclosure_d36c9b14-d61d-4487-bb23-51ca12744ec2_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_FinancialLiabilitiesFairValueDisclosure" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair value</link:label>
    <link:label id="lab_us-gaap_FinancialLiabilitiesFairValueDisclosure_label_en-US" xlink:label="lab_us-gaap_FinancialLiabilitiesFairValueDisclosure" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Liabilities Fair Value Disclosure</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancialLiabilitiesFairValueDisclosure" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancialLiabilitiesFairValueDisclosure"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancialLiabilitiesFairValueDisclosure" xlink:to="lab_us-gaap_FinancialLiabilitiesFairValueDisclosure" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentHoldingsOtherThanSecuritiesTable_60fa26a8-4bbd-4a78-b89c-cfe7af4861f5_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentHoldingsOtherThanSecuritiesTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Holdings, Other than Securities [Table]</link:label>
    <link:label id="lab_us-gaap_InvestmentHoldingsOtherThanSecuritiesTable_label_en-US" xlink:label="lab_us-gaap_InvestmentHoldingsOtherThanSecuritiesTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Holdings, Other than Securities [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentHoldingsOtherThanSecuritiesTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentHoldingsOtherThanSecuritiesTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentHoldingsOtherThanSecuritiesTable" xlink:to="lab_us-gaap_InvestmentHoldingsOtherThanSecuritiesTable" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_LocalPhoneNumber_aa11dc3b-6076-4c39-a821-e63c5e2bcc7e_terseLabel_en-US" xlink:label="lab_dei_LocalPhoneNumber" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Local Phone Number</link:label>
    <link:label id="lab_dei_LocalPhoneNumber_label_en-US" xlink:label="lab_dei_LocalPhoneNumber" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Local Phone Number</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_LocalPhoneNumber" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_LocalPhoneNumber"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_LocalPhoneNumber" xlink:to="lab_dei_LocalPhoneNumber" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RepurchaseAgreementsCost_c504c5f5-0452-4e8c-ab38-1ec9ab1549bb_terseLabel_en-US" xlink:label="lab_efc_RepurchaseAgreementsCost" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase agreements, cost</link:label>
    <link:label id="lab_efc_RepurchaseAgreementsCost_label_en-US" xlink:label="lab_efc_RepurchaseAgreementsCost" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase agreements, cost</link:label>
    <link:label id="lab_efc_RepurchaseAgreementsCost_documentation_en-US" xlink:label="lab_efc_RepurchaseAgreementsCost" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cost basis of reverse repurchase agreements carried at fair value.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RepurchaseAgreementsCost" xlink:href="efc-20201231.xsd#efc_RepurchaseAgreementsCost"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RepurchaseAgreementsCost" xlink:to="lab_efc_RepurchaseAgreementsCost" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeAssetsLiabilitiesAtFairValueNet_e54fade9-3693-4ceb-9ae9-7d71a8ddbc82_totalLabel_en-US" xlink:label="lab_us-gaap_DerivativeAssetsLiabilitiesAtFairValueNet" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total</link:label>
    <link:label id="lab_us-gaap_DerivativeAssetsLiabilitiesAtFairValueNet_db443c98-92b2-4d4e-ae51-a5ca631bdd55_negatedLabel_en-US" xlink:label="lab_us-gaap_DerivativeAssetsLiabilitiesAtFairValueNet" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value</link:label>
    <link:label id="lab_us-gaap_DerivativeAssetsLiabilitiesAtFairValueNet_93f1e593-5d62-444a-b606-e124616db5de_verboseLabel_en-US" xlink:label="lab_us-gaap_DerivativeAssetsLiabilitiesAtFairValueNet" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value</link:label>
    <link:label id="lab_us-gaap_DerivativeAssetsLiabilitiesAtFairValueNet_10bbacf3-0f78-4140-a239-46ce9a1b4e8e_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeAssetsLiabilitiesAtFairValueNet" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Assets (Liabilities), at Fair Value, Net</link:label>
    <link:label id="lab_us-gaap_DerivativeAssetsLiabilitiesAtFairValueNet_label_en-US" xlink:label="lab_us-gaap_DerivativeAssetsLiabilitiesAtFairValueNet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Assets (Liabilities), at Fair Value, Net</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeAssetsLiabilitiesAtFairValueNet" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeAssetsLiabilitiesAtFairValueNet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeAssetsLiabilitiesAtFairValueNet" xlink:to="lab_us-gaap_DerivativeAssetsLiabilitiesAtFairValueNet" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CatchupPremiumAmortizationAdjustment_97b7b2b2-5817-4786-8f6a-65174d3752b0_terseLabel_en-US" xlink:label="lab_efc_CatchupPremiumAmortizationAdjustment" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Catch-up premium amortization adjustment</link:label>
    <link:label id="lab_efc_CatchupPremiumAmortizationAdjustment_label_en-US" xlink:label="lab_efc_CatchupPremiumAmortizationAdjustment" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Catch-up Premium Amortization Adjustment</link:label>
    <link:label id="lab_efc_CatchupPremiumAmortizationAdjustment_documentation_en-US" xlink:label="lab_efc_CatchupPremiumAmortizationAdjustment" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">For securities whose cash flows vary depending on prepayments, an effective yield retroactive to the time of purchase is periodically recomputed based on actual prepayments and changes in projected prepayment activity, and a catch-up adjustment, or "Catch-up Premium Amortization Adjustment," is made to amortization to reflect the cumulative impact of the change in effective yield.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CatchupPremiumAmortizationAdjustment" xlink:href="efc-20201231.xsd#efc_CatchupPremiumAmortizationAdjustment"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CatchupPremiumAmortizationAdjustment" xlink:to="lab_efc_CatchupPremiumAmortizationAdjustment" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LoansMeasurementInputRecoveryAmount_ea84bcaa-50cd-44a6-9243-b487621a457b_terseLabel_en-US" xlink:label="lab_efc_LoansMeasurementInputRecoveryAmount" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans, Measurement Input, Recovery Amount</link:label>
    <link:label id="lab_efc_LoansMeasurementInputRecoveryAmount_label_en-US" xlink:label="lab_efc_LoansMeasurementInputRecoveryAmount" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans, Measurement Input, Recovery Amount</link:label>
    <link:label id="lab_efc_LoansMeasurementInputRecoveryAmount_documentation_en-US" xlink:label="lab_efc_LoansMeasurementInputRecoveryAmount" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans, Measurement Input, Recovery Amount</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LoansMeasurementInputRecoveryAmount" xlink:href="efc-20201231.xsd#efc_LoansMeasurementInputRecoveryAmount"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LoansMeasurementInputRecoveryAmount" xlink:to="lab_efc_LoansMeasurementInputRecoveryAmount" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OthersecuredfinancingatfairvalueProceeds_9ce82c49-74c0-45b3-a7ff-bae93b0a4209_terseLabel_en-US" xlink:label="lab_efc_OthersecuredfinancingatfairvalueProceeds" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other secured financing, at fair value-Proceeds</link:label>
    <link:label id="lab_efc_OthersecuredfinancingatfairvalueProceeds_label_en-US" xlink:label="lab_efc_OthersecuredfinancingatfairvalueProceeds" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other secured financing, at fair value-Proceeds</link:label>
    <link:label id="lab_efc_OthersecuredfinancingatfairvalueProceeds_documentation_en-US" xlink:label="lab_efc_OthersecuredfinancingatfairvalueProceeds" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds received from secured financing other than securities sold under agreements to repurchase and securities loaned for which the Company has elected the fair value option.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OthersecuredfinancingatfairvalueProceeds" xlink:href="efc-20201231.xsd#efc_OthersecuredfinancingatfairvalueProceeds"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OthersecuredfinancingatfairvalueProceeds" xlink:to="lab_efc_OthersecuredfinancingatfairvalueProceeds" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ClosingDateofSecuritizationTransactionDomain_7f2d0ce1-f4c0-48fd-bad6-4854a079f135_terseLabel_en-US" xlink:label="lab_efc_ClosingDateofSecuritizationTransactionDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Closing Date of Securitization Transaction [Domain]</link:label>
    <link:label id="lab_efc_ClosingDateofSecuritizationTransactionDomain_label_en-US" xlink:label="lab_efc_ClosingDateofSecuritizationTransactionDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Closing Date of Securitization Transaction [Domain]</link:label>
    <link:label id="lab_efc_ClosingDateofSecuritizationTransactionDomain_documentation_en-US" xlink:label="lab_efc_ClosingDateofSecuritizationTransactionDomain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Domain] for Closing Date of Securitization Transaction [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ClosingDateofSecuritizationTransactionDomain" xlink:href="efc-20201231.xsd#efc_ClosingDateofSecuritizationTransactionDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ClosingDateofSecuritizationTransactionDomain" xlink:to="lab_efc_ClosingDateofSecuritizationTransactionDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ShareConversion_a92c6024-fb1d-4672-9e61-04ee54bf1710_terseLabel_en-US" xlink:label="lab_efc_ShareConversion" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share conversion</link:label>
    <link:label id="lab_efc_ShareConversion_label_en-US" xlink:label="lab_efc_ShareConversion" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share Conversion</link:label>
    <link:label id="lab_efc_ShareConversion_documentation_en-US" xlink:label="lab_efc_ShareConversion" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share Conversion</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ShareConversion" xlink:href="efc-20201231.xsd#efc_ShareConversion"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ShareConversion" xlink:to="lab_efc_ShareConversion" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureUnrealizedGainsLosses_08281439-12bc-44f6-bfb6-b47a919961c7_terseLabel_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureUnrealizedGainsLosses" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized gains (losses) on real estate owned, net</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureUnrealizedGainsLosses_63c7f0ae-3620-43fa-a6aa-f37e87572672_negatedTerseLabel_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureUnrealizedGainsLosses" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized (gains) losses on real estate owned, net</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureUnrealizedGainsLosses_label_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureUnrealizedGainsLosses" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure, Unrealized Gains (Losses)</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureUnrealizedGainsLosses_documentation_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureUnrealizedGainsLosses" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure, Unrealized Gains (Losses)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RealEstateAcquiredThroughForeclosureUnrealizedGainsLosses" xlink:href="efc-20201231.xsd#efc_RealEstateAcquiredThroughForeclosureUnrealizedGainsLosses"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RealEstateAcquiredThroughForeclosureUnrealizedGainsLosses" xlink:to="lab_efc_RealEstateAcquiredThroughForeclosureUnrealizedGainsLosses" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueByLiabilityClassAxis_c93990a1-ee86-47dd-af92-3c9ea2dfc0e8_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueByLiabilityClassAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liability Class [Axis]</link:label>
    <link:label id="lab_us-gaap_FairValueByLiabilityClassAxis_label_en-US" xlink:label="lab_us-gaap_FairValueByLiabilityClassAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liability Class [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueByLiabilityClassAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueByLiabilityClassAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueByLiabilityClassAxis" xlink:to="lab_us-gaap_FairValueByLiabilityClassAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RealizedgainorlossoninvestmentsfinancialderivativesandforeigncurrencytransactionsexcludesFXoncash_5be94d86-b3b5-4b63-9ad1-71d799782602_negatedTerseLabel_en-US" xlink:label="lab_efc_RealizedgainorlossoninvestmentsfinancialderivativesandforeigncurrencytransactionsexcludesFXoncash" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized gain or loss on investments, financial derivatives, and foreign currency transactions (excludes FX on cash)</link:label>
    <link:label id="lab_efc_RealizedgainorlossoninvestmentsfinancialderivativesandforeigncurrencytransactionsexcludesFXoncash_cd2e14d6-6ae1-44b7-80fb-13379e9db5fd_terseLabel_en-US" xlink:label="lab_efc_RealizedgainorlossoninvestmentsfinancialderivativesandforeigncurrencytransactionsexcludesFXoncash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized gain or loss on investments, financial derivatives, and foreign currency transactions (excludes FX on cash)</link:label>
    <link:label id="lab_efc_RealizedgainorlossoninvestmentsfinancialderivativesandforeigncurrencytransactionsexcludesFXoncash_label_en-US" xlink:label="lab_efc_RealizedgainorlossoninvestmentsfinancialderivativesandforeigncurrencytransactionsexcludesFXoncash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized gain or loss on investments, financial derivatives, and foreign currency transactions (excludes FX on cash)</link:label>
    <link:label id="lab_efc_RealizedgainorlossoninvestmentsfinancialderivativesandforeigncurrencytransactionsexcludesFXoncash_documentation_en-US" xlink:label="lab_efc_RealizedgainorlossoninvestmentsfinancialderivativesandforeigncurrencytransactionsexcludesFXoncash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total realized gains or losses on investments, financial derivatives, and foreign currency transactions on investments and financial derivatives. Excludes foreign currency transactions relating to cash.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RealizedgainorlossoninvestmentsfinancialderivativesandforeigncurrencytransactionsexcludesFXoncash" xlink:href="efc-20201231.xsd#efc_RealizedgainorlossoninvestmentsfinancialderivativesandforeigncurrencytransactionsexcludesFXoncash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RealizedgainorlossoninvestmentsfinancialderivativesandforeigncurrencytransactionsexcludesFXoncash" xlink:to="lab_efc_RealizedgainorlossoninvestmentsfinancialderivativesandforeigncurrencytransactionsexcludesFXoncash" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PositionDomain_58c3ae1b-1a29-4784-b434-9bf3327de9af_terseLabel_en-US" xlink:label="lab_us-gaap_PositionDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Position [Domain]</link:label>
    <link:label id="lab_us-gaap_PositionDomain_label_en-US" xlink:label="lab_us-gaap_PositionDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Position [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PositionDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PositionDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PositionDomain" xlink:to="lab_us-gaap_PositionDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PositionAxis_14803335-0897-42dd-96e1-a7c52037c708_terseLabel_en-US" xlink:label="lab_us-gaap_PositionAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Position [Axis]</link:label>
    <link:label id="lab_us-gaap_PositionAxis_label_en-US" xlink:label="lab_us-gaap_PositionAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Position [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PositionAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PositionAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PositionAxis" xlink:to="lab_us-gaap_PositionAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_TransactionDomain_d7ea8de9-6b12-43e9-8ce6-22cc73c7b8e0_terseLabel_en-US" xlink:label="lab_us-gaap_TransactionDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transaction [Domain]</link:label>
    <link:label id="lab_us-gaap_TransactionDomain_label_en-US" xlink:label="lab_us-gaap_TransactionDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transaction [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_TransactionDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_TransactionDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_TransactionDomain" xlink:to="lab_us-gaap_TransactionDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeFairValueOfDerivativeNet_7afdc3e0-149e-4aac-a3ef-64e290a4cff2_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeFairValueOfDerivativeNet" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value</link:label>
    <link:label id="lab_us-gaap_DerivativeFairValueOfDerivativeNet_label_en-US" xlink:label="lab_us-gaap_DerivativeFairValueOfDerivativeNet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative, Fair Value, Net</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeFairValueOfDerivativeNet" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeFairValueOfDerivativeNet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeFairValueOfDerivativeNet" xlink:to="lab_us-gaap_DerivativeFairValueOfDerivativeNet" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CouponRateOnUnderlyingCollateralOnRepurchaseAgreement_eadcbfc8-9a7c-441b-b295-15e6e073339a_terseLabel_en-US" xlink:label="lab_efc_CouponRateOnUnderlyingCollateralOnRepurchaseAgreement" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Coupon Rate On Underlying Collateral On Repurchase Agreement</link:label>
    <link:label id="lab_efc_CouponRateOnUnderlyingCollateralOnRepurchaseAgreement_label_en-US" xlink:label="lab_efc_CouponRateOnUnderlyingCollateralOnRepurchaseAgreement" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Coupon Rate On Underlying Collateral On Repurchase Agreement</link:label>
    <link:label id="lab_efc_CouponRateOnUnderlyingCollateralOnRepurchaseAgreement_documentation_en-US" xlink:label="lab_efc_CouponRateOnUnderlyingCollateralOnRepurchaseAgreement" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Coupon Rate On Underlying Collateral On Repurchase Agreement</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CouponRateOnUnderlyingCollateralOnRepurchaseAgreement" xlink:href="efc-20201231.xsd#efc_CouponRateOnUnderlyingCollateralOnRepurchaseAgreement"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CouponRateOnUnderlyingCollateralOnRepurchaseAgreement" xlink:to="lab_efc_CouponRateOnUnderlyingCollateralOnRepurchaseAgreement" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2022Member_db503556-4dde-4a05-a973-1c8876bf1e8a_terseLabel_en-US" xlink:label="lab_efc_A2022Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2022 [Member]</link:label>
    <link:label id="lab_efc_A2022Member_label_en-US" xlink:label="lab_efc_A2022Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2022 [Member]</link:label>
    <link:label id="lab_efc_A2022Member_documentation_en-US" xlink:label="lab_efc_A2022Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2022 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2022Member" xlink:href="efc-20201231.xsd#efc_A2022Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2022Member" xlink:to="lab_efc_A2022Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OperatingPartnershipUnitsconvertedduringperiod_cbd2f469-7d2b-46fb-ae26-cacfb41f10a1_terseLabel_en-US" xlink:label="lab_efc_OperatingPartnershipUnitsconvertedduringperiod" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership Units, converted during period</link:label>
    <link:label id="lab_efc_OperatingPartnershipUnitsconvertedduringperiod_label_en-US" xlink:label="lab_efc_OperatingPartnershipUnitsconvertedduringperiod" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership Units, converted during period</link:label>
    <link:label id="lab_efc_OperatingPartnershipUnitsconvertedduringperiod_documentation_en-US" xlink:label="lab_efc_OperatingPartnershipUnitsconvertedduringperiod" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The number of operating partnership units (as defined by the Company) converted to common shares during the period.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OperatingPartnershipUnitsconvertedduringperiod" xlink:href="efc-20201231.xsd#efc_OperatingPartnershipUnitsconvertedduringperiod"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OperatingPartnershipUnitsconvertedduringperiod" xlink:to="lab_efc_OperatingPartnershipUnitsconvertedduringperiod" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Upfrontpointsonwrittencreditderivative_4967bd1a-0aba-40b2-8657-b425e001ca8d_terseLabel_en-US" xlink:label="lab_efc_Upfrontpointsonwrittencreditderivative" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Upfront points on written credit derivative in excess of threshold</link:label>
    <link:label id="lab_efc_Upfrontpointsonwrittencreditderivative_label_en-US" xlink:label="lab_efc_Upfrontpointsonwrittencreditderivative" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Upfront points on written credit derivative</link:label>
    <link:label id="lab_efc_Upfrontpointsonwrittencreditderivative_documentation_en-US" xlink:label="lab_efc_Upfrontpointsonwrittencreditderivative" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Estimated upfront points on written credit derivative contract at period end for those contracts where the current spread is in excess of a predetermined threshold.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Upfrontpointsonwrittencreditderivative" xlink:href="efc-20201231.xsd#efc_Upfrontpointsonwrittencreditderivative"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Upfrontpointsonwrittencreditderivative" xlink:to="lab_efc_Upfrontpointsonwrittencreditderivative" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesLineItems_0cf1fa26-02f9-4348-80d0-d0c24e3d7cfc_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Measurements Inputs and Valuation Techniques [Line Items]</link:label>
    <link:label id="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesLineItems_label_en-US" xlink:label="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Measurement Inputs and Valuation Techniques [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesLineItems" xlink:to="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OtherAssets_2c73cec0-b87e-4019-a556-57543130328d_terseLabel_en-US" xlink:label="lab_us-gaap_OtherAssets" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other assets</link:label>
    <link:label id="lab_us-gaap_OtherAssets_label_en-US" xlink:label="lab_us-gaap_OtherAssets" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Assets</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OtherAssets" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OtherAssets"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OtherAssets" xlink:to="lab_us-gaap_OtherAssets" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_GuarantyLiabilities_6e01f065-0194-4c2c-a987-fd76b07e7a3a_terseLabel_en-US" xlink:label="lab_us-gaap_GuarantyLiabilities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Guaranty Liabilities</link:label>
    <link:label id="lab_us-gaap_GuarantyLiabilities_label_en-US" xlink:label="lab_us-gaap_GuarantyLiabilities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Guaranty Liabilities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_GuarantyLiabilities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_GuarantyLiabilities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_GuarantyLiabilities" xlink:to="lab_us-gaap_GuarantyLiabilities" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseFairValueOfCollateral_e2c417dd-cada-4626-8ce2-f3895027a1c3_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseFairValueOfCollateral" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Collateral relating to Reverse Repurchase Agreements</link:label>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseFairValueOfCollateral_label_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseFairValueOfCollateral" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Sold under Agreements to Repurchase, Fair Value of Collateral</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseFairValueOfCollateral" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseFairValueOfCollateral"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseFairValueOfCollateral" xlink:to="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseFairValueOfCollateral" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetsChangeInNetUnrealizedGainLoss_7ee96e44-5eb8-4df2-9002-94a95eddf472_terseLabel_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetsChangeInNetUnrealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets, Change In Net Unrealized Gain/(Loss)</link:label>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetsChangeInNetUnrealizedGainLoss_label_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetsChangeInNetUnrealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement With Unobservable Inputs Reconciliation, Recurring Basis, Assets, Change In Net Unrealized Gain/(Loss)</link:label>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetsChangeInNetUnrealizedGainLoss_documentation_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetsChangeInNetUnrealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Change in unrealized gain/loss on investment/derivatives classed as Level 3 within the fair value heirarchy.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetsChangeInNetUnrealizedGainLoss" xlink:href="efc-20201231.xsd#efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetsChangeInNetUnrealizedGainLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetsChangeInNetUnrealizedGainLoss" xlink:to="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetsChangeInNetUnrealizedGainLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year4Point50PercentMember_626d2137-ba5b-4603-8ed2-e1efd701db4e_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year4Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 4 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year4Point50PercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year4Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 4 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year4Point50PercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year4Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 4.50%</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociation30Year4Point50PercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociation30Year4Point50PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociation30Year4Point50PercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociation30Year4Point50PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePointFivePercentMember_bdc38938-1c94-4fb1-8583-a244a26ced7d_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Three Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Three Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Three Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePointFivePercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised_215586dd-8d32-4dc5-8676-70e3cd432d25_terseLabel_en-US" xlink:label="lab_us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Compensation Arrangement by Share-based Payment Award, Options, Exercises in Period</link:label>
    <link:label id="lab_us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised_528ea911-6b28-4b5d-a0a0-4390c672a1e8_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Exercised</link:label>
    <link:label id="lab_us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised_label_en-US" xlink:label="lab_us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Compensation Arrangement by Share-based Payment Award, Options, Exercises in Period</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised" xlink:to="lab_us-gaap_StockIssuedDuringPeriodSharesStockOptionsExercised" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriod_229be965-124c-4535-bf1d-52a8fd89b3ab_terseLabel_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriod" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">LTIP Unit Grants in Period (in units)</link:label>
    <link:label id="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriod_label_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriod" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Compensation Arrangement by Share-based Payment Award, Options, Grants in Period, Net of Forfeitures</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriod" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriod"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriod" xlink:to="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriod" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LoanaccrualstatusAxis_6810ff20-afab-4bca-a6c3-25fa159c532b_terseLabel_en-US" xlink:label="lab_efc_LoanaccrualstatusAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loan accrual status [Axis]</link:label>
    <link:label id="lab_efc_LoanaccrualstatusAxis_label_en-US" xlink:label="lab_efc_LoanaccrualstatusAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loan accrual status [Axis]</link:label>
    <link:label id="lab_efc_LoanaccrualstatusAxis_documentation_en-US" xlink:label="lab_efc_LoanaccrualstatusAxis" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loan accrual status [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LoanaccrualstatusAxis" xlink:href="efc-20201231.xsd#efc_LoanaccrualstatusAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LoanaccrualstatusAxis" xlink:to="lab_efc_LoanaccrualstatusAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PreferredStockDividendRatePercentage_0d2114d2-5732-4d3d-ab3e-5cbd059855a4_terseLabel_en-US" xlink:label="lab_us-gaap_PreferredStockDividendRatePercentage" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Dividend Rate, Percentage</link:label>
    <link:label id="lab_us-gaap_PreferredStockDividendRatePercentage_label_en-US" xlink:label="lab_us-gaap_PreferredStockDividendRatePercentage" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Dividend Rate, Percentage</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PreferredStockDividendRatePercentage" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PreferredStockDividendRatePercentage"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PreferredStockDividendRatePercentage" xlink:to="lab_us-gaap_PreferredStockDividendRatePercentage" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativesPolicyTextBlock_5d002d33-4950-4d7e-8c0f-c1290fd2e58b_verboseLabel_en-US" xlink:label="lab_us-gaap_DerivativesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Derivatives</link:label>
    <link:label id="lab_us-gaap_DerivativesPolicyTextBlock_label_en-US" xlink:label="lab_us-gaap_DerivativesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivatives, Policy [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativesPolicyTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativesPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativesPolicyTextBlock" xlink:to="lab_us-gaap_DerivativesPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_GeneralInsuranceExpense_d6e6fffa-cced-43ba-b744-c5ce61b93019_terseLabel_en-US" xlink:label="lab_us-gaap_GeneralInsuranceExpense" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">General Insurance Expense</link:label>
    <link:label id="lab_us-gaap_GeneralInsuranceExpense_label_en-US" xlink:label="lab_us-gaap_GeneralInsuranceExpense" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">General Insurance Expense</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_GeneralInsuranceExpense" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_GeneralInsuranceExpense"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_GeneralInsuranceExpense" xlink:to="lab_us-gaap_GeneralInsuranceExpense" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InvestmentsinOperatingEntitiesPolicyTextBlock_deb27855-fa93-45b9-8866-7ea1d9aa0a85_terseLabel_en-US" xlink:label="lab_efc_InvestmentsinOperatingEntitiesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments in Operating Entities</link:label>
    <link:label id="lab_efc_InvestmentsinOperatingEntitiesPolicyTextBlock_label_en-US" xlink:label="lab_efc_InvestmentsinOperatingEntitiesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments in Operating Entities [Policy Text Block]</link:label>
    <link:label id="lab_efc_InvestmentsinOperatingEntitiesPolicyTextBlock_documentation_en-US" xlink:label="lab_efc_InvestmentsinOperatingEntitiesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The Company's accounting policy for investments in operating entities.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InvestmentsinOperatingEntitiesPolicyTextBlock" xlink:href="efc-20201231.xsd#efc_InvestmentsinOperatingEntitiesPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InvestmentsinOperatingEntitiesPolicyTextBlock" xlink:to="lab_efc_InvestmentsinOperatingEntitiesPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporation15Year3PercentMember_461920b9-fb91-4464-9cf8-18fe98857189_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporation15Year3PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation (15 Year) 3 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporation15Year3PercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporation15Year3PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation (15 Year) 3 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporation15Year3PercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporation15Year3PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation (15 Year) 3.00%</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporation15Year3PercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporation15Year3PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporation15Year3PercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporation15Year3PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MinimumRequiredRegulatoryBalanceMember_2e64a014-76e8-461b-95be-c3e4b0d885b4_terseLabel_en-US" xlink:label="lab_efc_MinimumRequiredRegulatoryBalanceMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Minimum Required Regulatory Balance [Member]</link:label>
    <link:label id="lab_efc_MinimumRequiredRegulatoryBalanceMember_label_en-US" xlink:label="lab_efc_MinimumRequiredRegulatoryBalanceMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Minimum Required Regulatory Balance [Member]</link:label>
    <link:label id="lab_efc_MinimumRequiredRegulatoryBalanceMember_documentation_en-US" xlink:label="lab_efc_MinimumRequiredRegulatoryBalanceMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Minimum Required Regulatory Balance [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MinimumRequiredRegulatoryBalanceMember" xlink:href="efc-20201231.xsd#efc_MinimumRequiredRegulatoryBalanceMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MinimumRequiredRegulatoryBalanceMember" xlink:to="lab_efc_MinimumRequiredRegulatoryBalanceMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NotePurchaseAgreementMay2019Member_b570af25-3ec1-4c3e-b28d-fc3f6639dd82_terseLabel_en-US" xlink:label="lab_efc_NotePurchaseAgreementMay2019Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Note Purchase Agreement May 2019 [Member]</link:label>
    <link:label id="lab_efc_NotePurchaseAgreementMay2019Member_label_en-US" xlink:label="lab_efc_NotePurchaseAgreementMay2019Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Note Purchase Agreement May 2019 [Member]</link:label>
    <link:label id="lab_efc_NotePurchaseAgreementMay2019Member_documentation_en-US" xlink:label="lab_efc_NotePurchaseAgreementMay2019Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Note purchase agreement (agreement to lend) executed in May 2019.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NotePurchaseAgreementMay2019Member" xlink:href="efc-20201231.xsd#efc_NotePurchaseAgreementMay2019Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NotePurchaseAgreementMay2019Member" xlink:to="lab_efc_NotePurchaseAgreementMay2019Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NetRealizedGainLossonDerivativesincludinggainlossonforeigncurrencytransactions_dc26930f-32f9-4936-a847-fa6041f5e161_terseLabel_en-US" xlink:label="lab_efc_NetRealizedGainLossonDerivativesincludinggainlossonforeigncurrencytransactions" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized gains (losses) on financial derivatives, net</link:label>
    <link:label id="lab_efc_NetRealizedGainLossonDerivativesincludinggainlossonforeigncurrencytransactions_label_en-US" xlink:label="lab_efc_NetRealizedGainLossonDerivativesincludinggainlossonforeigncurrencytransactions" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Realized Gain/Loss on Derivatives including gain (loss) on foreign currency transactions</link:label>
    <link:label id="lab_efc_NetRealizedGainLossonDerivativesincludinggainlossonforeigncurrencytransactions_documentation_en-US" xlink:label="lab_efc_NetRealizedGainLossonDerivativesincludinggainlossonforeigncurrencytransactions" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Realized Gain/Loss on Derivatives including gain (loss) on foreign currency transactions</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NetRealizedGainLossonDerivativesincludinggainlossonforeigncurrencytransactions" xlink:href="efc-20201231.xsd#efc_NetRealizedGainLossonDerivativesincludinggainlossonforeigncurrencytransactions"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NetRealizedGainLossonDerivativesincludinggainlossonforeigncurrencytransactions" xlink:to="lab_efc_NetRealizedGainLossonDerivativesincludinggainlossonforeigncurrencytransactions" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeAssetNotOffsetPolicyElectionDeduction_3e2991fb-16e6-4497-9804-6755881b0d52_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeAssetNotOffsetPolicyElectionDeduction" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instruments Available for Offset</link:label>
    <link:label id="lab_us-gaap_DerivativeAssetNotOffsetPolicyElectionDeduction_label_en-US" xlink:label="lab_us-gaap_DerivativeAssetNotOffsetPolicyElectionDeduction" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Asset, Not Offset, Policy Election Deduction</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeAssetNotOffsetPolicyElectionDeduction" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeAssetNotOffsetPolicyElectionDeduction"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeAssetNotOffsetPolicyElectionDeduction" xlink:to="lab_us-gaap_DerivativeAssetNotOffsetPolicyElectionDeduction" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeAssetMeasurementInput_eadeda1e-6452-4597-bdbc-b55e24df7f6a_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeAssetMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative asset, measurement input</link:label>
    <link:label id="lab_us-gaap_DerivativeAssetMeasurementInput_label_en-US" xlink:label="lab_us-gaap_DerivativeAssetMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Asset, Measurement Input</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeAssetMeasurementInput" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeAssetMeasurementInput"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeAssetMeasurementInput" xlink:to="lab_us-gaap_DerivativeAssetMeasurementInput" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_GovernmentSectorMember_03e48c2a-4e23-4c62-a0f8-3429d890586d_terseLabel_en-US" xlink:label="lab_us-gaap_GovernmentSectorMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government Sector [Member]</link:label>
    <link:label id="lab_us-gaap_GovernmentSectorMember_label_en-US" xlink:label="lab_us-gaap_GovernmentSectorMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government Sector [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_GovernmentSectorMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_GovernmentSectorMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_GovernmentSectorMember" xlink:to="lab_us-gaap_GovernmentSectorMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect_7cf0e824-032c-43ef-bfb8-dfb5cc0ed0b6_totalLabel_en-US" xlink:label="lab_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Increase (Decrease) in Cash, Cash Equivalents, and Restricted Cash</link:label>
    <link:label id="lab_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect_label_en-US" xlink:label="lab_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash, Cash Equivalents, Restricted Cash and Restricted Cash Equivalents, Period Increase (Decrease), Including Exchange Rate Effect</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect" xlink:to="lab_us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Unrealizedgainslossesothernet_98270ebf-eeb3-4a9f-89de-ba25bbd6f046_negatedTerseLabel_en-US" xlink:label="lab_efc_Unrealizedgainslossesothernet" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized (gains) losses other, net</link:label>
    <link:label id="lab_efc_Unrealizedgainslossesothernet_label_en-US" xlink:label="lab_efc_Unrealizedgainslossesothernet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized gains (losses) other, net</link:label>
    <link:label id="lab_efc_Unrealizedgainslossesothernet_documentation_en-US" xlink:label="lab_efc_Unrealizedgainslossesothernet" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized gains (losses) other, net</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Unrealizedgainslossesothernet" xlink:href="efc-20201231.xsd#efc_Unrealizedgainslossesothernet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Unrealizedgainslossesothernet" xlink:to="lab_efc_Unrealizedgainslossesothernet" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluefinancingactivities_5fca6246-621f-4a31-bc29-c4dadaa00593_terseLabel_en-US" xlink:label="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluefinancingactivities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt issuance costs related to Other secured borrowings at fair value financing activities</link:label>
    <link:label id="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluefinancingactivities_label_en-US" xlink:label="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluefinancingactivities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt issuance costs related to Other secured borrowings at fair value financing activities</link:label>
    <link:label id="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluefinancingactivities_documentation_en-US" xlink:label="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluefinancingactivities" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt issuance costs related to Other secured borrowings at fair value financing activities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluefinancingactivities" xlink:href="efc-20201231.xsd#efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluefinancingactivities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluefinancingactivities" xlink:to="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluefinancingactivities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue13600U.S.TreasuryNoteCoupon2.88MaturityDate202111Member_42b20c1d-ad9e-48e8-a899-558dd7b5a836_terseLabel_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue13600U.S.TreasuryNoteCoupon2.88MaturityDate202111Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $13,600 U.S. Treasury Note, Coupon 2.88%, Maturity Date 2021 11 [Member]</link:label>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue13600U.S.TreasuryNoteCoupon2.88MaturityDate202111Member_label_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue13600U.S.TreasuryNoteCoupon2.88MaturityDate202111Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $13,600 U.S. Treasury Note, Coupon 2.88%, Maturity Date 2021 11 [Member]</link:label>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue13600U.S.TreasuryNoteCoupon2.88MaturityDate202111Member_documentation_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue13600U.S.TreasuryNoteCoupon2.88MaturityDate202111Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $13,600 U.S. Treasury Note, Coupon 2.88%, Maturity Date 2021 11 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_JPMorganSecuritiesLLCCollateralizedbyParValue13600U.S.TreasuryNoteCoupon2.88MaturityDate202111Member" xlink:href="efc-20201231.xsd#efc_JPMorganSecuritiesLLCCollateralizedbyParValue13600U.S.TreasuryNoteCoupon2.88MaturityDate202111Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_JPMorganSecuritiesLLCCollateralizedbyParValue13600U.S.TreasuryNoteCoupon2.88MaturityDate202111Member" xlink:to="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue13600U.S.TreasuryNoteCoupon2.88MaturityDate202111Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritizedResidentialMortgageLoansMember_a20eb640-d500-4002-982d-f13eeceeb6e3_terseLabel_en-US" xlink:label="lab_efc_SecuritizedResidentialMortgageLoansMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitized residential mortgage loans</link:label>
    <link:label id="lab_efc_SecuritizedResidentialMortgageLoansMember_label_en-US" xlink:label="lab_efc_SecuritizedResidentialMortgageLoansMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitized Residential Mortgage Loans [Member]</link:label>
    <link:label id="lab_efc_SecuritizedResidentialMortgageLoansMember_documentation_en-US" xlink:label="lab_efc_SecuritizedResidentialMortgageLoansMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitized Residential Mortgage Loans [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritizedResidentialMortgageLoansMember" xlink:href="efc-20201231.xsd#efc_SecuritizedResidentialMortgageLoansMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritizedResidentialMortgageLoansMember" xlink:to="lab_efc_SecuritizedResidentialMortgageLoansMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationNetIncomeLoss_4a2f38ed-c5ae-4d86-aa20-d8f6088edfc6_terseLabel_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationNetIncomeLoss" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Net Income (Loss)</link:label>
    <link:label id="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationNetIncomeLoss_label_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationNetIncomeLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Net Income (Loss)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationNetIncomeLoss" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquityMethodInvestmentSummarizedFinancialInformationNetIncomeLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationNetIncomeLoss" xlink:to="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationNetIncomeLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritiesPurchasedunderAgreementstoResellMember_d25b687b-307f-4259-ab7e-95786ce1a941_terseLabel_en-US" xlink:label="lab_efc_SecuritiesPurchasedunderAgreementstoResellMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Purchased under Agreements to Resell [Member]</link:label>
    <link:label id="lab_efc_SecuritiesPurchasedunderAgreementstoResellMember_label_en-US" xlink:label="lab_efc_SecuritiesPurchasedunderAgreementstoResellMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Purchased under Agreements to Resell [Member]</link:label>
    <link:label id="lab_efc_SecuritiesPurchasedunderAgreementstoResellMember_documentation_en-US" xlink:label="lab_efc_SecuritiesPurchasedunderAgreementstoResellMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities purchased with a simultaneous agreement to resell (the identical or substantially the same securities) at a specified date for a specified price.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritiesPurchasedunderAgreementstoResellMember" xlink:href="efc-20201231.xsd#efc_SecuritiesPurchasedunderAgreementstoResellMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritiesPurchasedunderAgreementstoResellMember" xlink:to="lab_efc_SecuritiesPurchasedunderAgreementstoResellMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NetIncomeLoss_503f2108-f6cc-4ce0-8369-a5ff947dc930_totalLabel_en-US" xlink:label="lab_us-gaap_NetIncomeLoss" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Income (Loss) Attributable to Common Stockholders</link:label>
    <link:label id="lab_us-gaap_NetIncomeLoss_08153964-33f9-4b7b-9994-43733a414254_terseLabel_en-US" xlink:label="lab_us-gaap_NetIncomeLoss" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net income (loss) attributable to common stockholders</link:label>
    <link:label id="lab_us-gaap_NetIncomeLoss_label_en-US" xlink:label="lab_us-gaap_NetIncomeLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Income (Loss) Attributable to Parent</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NetIncomeLoss" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NetIncomeLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NetIncomeLoss" xlink:to="lab_us-gaap_NetIncomeLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Interestexpenserelatedtoconsolidatedsecuritizationtrust_d07cdf9f-492e-4c86-bcde-293f84e5408c_terseLabel_en-US" xlink:label="lab_efc_Interestexpenserelatedtoconsolidatedsecuritizationtrust" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest expense related to consolidated securitization trust</link:label>
    <link:label id="lab_efc_Interestexpenserelatedtoconsolidatedsecuritizationtrust_label_en-US" xlink:label="lab_efc_Interestexpenserelatedtoconsolidatedsecuritizationtrust" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest expense related to consolidated securitization trust</link:label>
    <link:label id="lab_efc_Interestexpenserelatedtoconsolidatedsecuritizationtrust_documentation_en-US" xlink:label="lab_efc_Interestexpenserelatedtoconsolidatedsecuritizationtrust" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest expense related to consolidated securitization trust</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Interestexpenserelatedtoconsolidatedsecuritizationtrust" xlink:href="efc-20201231.xsd#efc_Interestexpenserelatedtoconsolidatedsecuritizationtrust"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Interestexpenserelatedtoconsolidatedsecuritizationtrust" xlink:to="lab_efc_Interestexpenserelatedtoconsolidatedsecuritizationtrust" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritizationTransactionsTextBlock_09a1640f-c9b3-447e-a5f6-25422c290760_terseLabel_en-US" xlink:label="lab_efc_SecuritizationTransactionsTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization Transactions</link:label>
    <link:label id="lab_efc_SecuritizationTransactionsTextBlock_label_en-US" xlink:label="lab_efc_SecuritizationTransactionsTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization Transactions [Text Block]</link:label>
    <link:label id="lab_efc_SecuritizationTransactionsTextBlock_documentation_en-US" xlink:label="lab_efc_SecuritizationTransactionsTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disclosure of securitization transactions for which the Company is a party to. Can include transactions accounted for as sales as well as transactions not accounted for as sales. Includes information on the nature of the transactions, the Company's involvement and its consolidation conclusions.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritizationTransactionsTextBlock" xlink:href="efc-20201231.xsd#efc_SecuritizationTransactionsTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritizationTransactionsTextBlock" xlink:to="lab_efc_SecuritizationTransactionsTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CreditDefaultSwapsOnAssetBackedIndicesMember_8411b79d-2a7e-4cd0-a587-ef3c0342d3ba_terseLabel_en-US" xlink:label="lab_efc_CreditDefaultSwapsOnAssetBackedIndicesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit default swaps on asset-backed indices</link:label>
    <link:label id="lab_efc_CreditDefaultSwapsOnAssetBackedIndicesMember_label_en-US" xlink:label="lab_efc_CreditDefaultSwapsOnAssetBackedIndicesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swaps On Asset Backed Indices [Member]</link:label>
    <link:label id="lab_efc_CreditDefaultSwapsOnAssetBackedIndicesMember_documentation_en-US" xlink:label="lab_efc_CreditDefaultSwapsOnAssetBackedIndicesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">A type of swap transaction used as a credit derivative in which one party pays periodic payments to a counterparty and is guaranteed the promise of a pay-off in the event of a default over a basket of asset backed securities.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CreditDefaultSwapsOnAssetBackedIndicesMember" xlink:href="efc-20201231.xsd#efc_CreditDefaultSwapsOnAssetBackedIndicesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CreditDefaultSwapsOnAssetBackedIndicesMember" xlink:to="lab_efc_CreditDefaultSwapsOnAssetBackedIndicesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPointFivePercentMember_7216d14e-379a-4777-b8da-a1bfb4b278bb_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Fifteen Year) Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Fifteen Year) Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Fifteen Year) Four Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPointFivePercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisChangeInUnrealizedGainLoss_0d56ccd3-2efb-4a39-a97e-5e42a4c8d61e_verboseLabel_en-US" xlink:label="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisChangeInUnrealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Change in unrealized gains (losses), liabilities</link:label>
    <link:label id="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisChangeInUnrealizedGainLoss_bcb48937-98fb-40d3-98ca-1d198f8d34ad_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisChangeInUnrealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Change in unrealized gains (losses)</link:label>
    <link:label id="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisChangeInUnrealizedGainLoss_label_en-US" xlink:label="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisChangeInUnrealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Liability, Recurring Basis, Still Held, Unrealized Gain (Loss)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisChangeInUnrealizedGainLoss" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisChangeInUnrealizedGainLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisChangeInUnrealizedGainLoss" xlink:to="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisChangeInUnrealizedGainLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PrincipalAndInterestFloatingRateAgencySecuritiesMember_c7ce5ebd-9b08-4206-a2fe-ccfb33379193_terseLabel_en-US" xlink:label="lab_efc_PrincipalAndInterestFloatingRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal And Interest Floating Rate Agency Securities [Member]</link:label>
    <link:label id="lab_efc_PrincipalAndInterestFloatingRateAgencySecuritiesMember_label_en-US" xlink:label="lab_efc_PrincipalAndInterestFloatingRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal And Interest Floating Rate Agency Securities [Member]</link:label>
    <link:label id="lab_efc_PrincipalAndInterestFloatingRateAgencySecuritiesMember_documentation_en-US" xlink:label="lab_efc_PrincipalAndInterestFloatingRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">A security issued by a government agency or government sponsored enterprise which makes principal and variable interest rate payments.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PrincipalAndInterestFloatingRateAgencySecuritiesMember" xlink:href="efc-20201231.xsd#efc_PrincipalAndInterestFloatingRateAgencySecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PrincipalAndInterestFloatingRateAgencySecuritiesMember" xlink:to="lab_efc_PrincipalAndInterestFloatingRateAgencySecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncomeTaxesPaid_a0c035ff-de92-43ed-be57-59726d23ad90_terseLabel_en-US" xlink:label="lab_us-gaap_IncomeTaxesPaid" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Income tax paid</link:label>
    <link:label id="lab_us-gaap_IncomeTaxesPaid_label_en-US" xlink:label="lab_us-gaap_IncomeTaxesPaid" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Income Taxes Paid</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncomeTaxesPaid" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncomeTaxesPaid"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncomeTaxesPaid" xlink:to="lab_us-gaap_IncomeTaxesPaid" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LongInvestmentAaRatingMember_e1e76bb9-25ea-4e66-8cd5-6fbf51cc4032_terseLabel_en-US" xlink:label="lab_efc_LongInvestmentAaRatingMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Aa Rating [Member]</link:label>
    <link:label id="lab_efc_LongInvestmentAaRatingMember_label_en-US" xlink:label="lab_efc_LongInvestmentAaRatingMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Aa Rating [Member]</link:label>
    <link:label id="lab_efc_LongInvestmentAaRatingMember_documentation_en-US" xlink:label="lab_efc_LongInvestmentAaRatingMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Aa Rating [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LongInvestmentAaRatingMember" xlink:href="efc-20201231.xsd#efc_LongInvestmentAaRatingMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LongInvestmentAaRatingMember" xlink:to="lab_efc_LongInvestmentAaRatingMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationNetinterestincome_4a3bedac-b1c9-4077-a158-3a96e2791e7d_terseLabel_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationNetinterestincome" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Net interest income</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationNetinterestincome_label_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationNetinterestincome" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Net interest income</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationNetinterestincome_documentation_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationNetinterestincome" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount of interest income net of interest expense reported by equity method investee.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationNetinterestincome" xlink:href="efc-20201231.xsd#efc_EquityMethodInvestmentSummarizedFinancialInformationNetinterestincome"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationNetinterestincome" xlink:to="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationNetinterestincome" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtPolicyTextBlock_3038dfc7-2acb-4a43-b295-6124d2ab1ba8_terseLabel_en-US" xlink:label="lab_us-gaap_DebtPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Issuance Costs</link:label>
    <link:label id="lab_us-gaap_DebtPolicyTextBlock_label_en-US" xlink:label="lab_us-gaap_DebtPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt, Policy [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtPolicyTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtPolicyTextBlock" xlink:to="lab_us-gaap_DebtPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleofTBAsecuritiesTableTextBlock_8e87181b-5a58-4c1e-a3bd-ce3bae2ff45c_terseLabel_en-US" xlink:label="lab_efc_ScheduleofTBAsecuritiesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of TBA securities [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleofTBAsecuritiesTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleofTBAsecuritiesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of TBA securities [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleofTBAsecuritiesTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleofTBAsecuritiesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Tabular disclosure of TBA securities, classified as derivatives, including notional amount, cost basis, market value, and net carrying cost.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleofTBAsecuritiesTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleofTBAsecuritiesTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleofTBAsecuritiesTableTextBlock" xlink:to="lab_efc_ScheduleofTBAsecuritiesTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueBalanceSheetGroupingFinancialStatementCaptionsLineItems_bf038931-76e6-4e10-88d7-aa147c2bd38c_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueBalanceSheetGroupingFinancialStatementCaptionsLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Balance Sheet Grouping, Financial Statement Captions [Line Items]</link:label>
    <link:label id="lab_us-gaap_FairValueBalanceSheetGroupingFinancialStatementCaptionsLineItems_label_en-US" xlink:label="lab_us-gaap_FairValueBalanceSheetGroupingFinancialStatementCaptionsLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Balance Sheet Grouping, Financial Statement Captions [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueBalanceSheetGroupingFinancialStatementCaptionsLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueBalanceSheetGroupingFinancialStatementCaptionsLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueBalanceSheetGroupingFinancialStatementCaptionsLineItems" xlink:to="lab_us-gaap_FairValueBalanceSheetGroupingFinancialStatementCaptionsLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InterestOnlyFixedRateAgencySecuritiesMember_4dd10db0-7334-458f-9158-11409f4fde67_terseLabel_en-US" xlink:label="lab_efc_InterestOnlyFixedRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Only Fixed Rate Agency Securities [Member]</link:label>
    <link:label id="lab_efc_InterestOnlyFixedRateAgencySecuritiesMember_label_en-US" xlink:label="lab_efc_InterestOnlyFixedRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Only Fixed Rate Agency Securities [Member]</link:label>
    <link:label id="lab_efc_InterestOnlyFixedRateAgencySecuritiesMember_documentation_en-US" xlink:label="lab_efc_InterestOnlyFixedRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">A security issued by a government agency or government sponsored enterprise which makes fixed interest rate payments.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InterestOnlyFixedRateAgencySecuritiesMember" xlink:href="efc-20201231.xsd#efc_InterestOnlyFixedRateAgencySecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InterestOnlyFixedRateAgencySecuritiesMember" xlink:to="lab_efc_InterestOnlyFixedRateAgencySecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquityMethodInvestmentsMember_bdbded60-3f99-45cb-839a-b4d6837f42c5_terseLabel_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment in unconsolidated entities</link:label>
    <link:label id="lab_us-gaap_EquityMethodInvestmentsMember_label_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investments [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquityMethodInvestmentsMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquityMethodInvestmentsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquityMethodInvestmentsMember" xlink:to="lab_us-gaap_EquityMethodInvestmentsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CommitmentsAndContingenciesDisclosureAbstract_05aab1c1-672d-4616-967a-5c52ca212fd1_terseLabel_en-US" xlink:label="lab_us-gaap_CommitmentsAndContingenciesDisclosureAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commitments and Contingencies Disclosure [Abstract]</link:label>
    <link:label id="lab_us-gaap_CommitmentsAndContingenciesDisclosureAbstract_label_en-US" xlink:label="lab_us-gaap_CommitmentsAndContingenciesDisclosureAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commitments and Contingencies Disclosure [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CommitmentsAndContingenciesDisclosureAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CommitmentsAndContingenciesDisclosureAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CommitmentsAndContingenciesDisclosureAbstract" xlink:to="lab_us-gaap_CommitmentsAndContingenciesDisclosureAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_StatementScenarioAxis_dd7e695c-3401-4c28-bcdc-21722dfab17c_terseLabel_en-US" xlink:label="lab_srt_StatementScenarioAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Scenario [Axis]</link:label>
    <link:label id="lab_srt_StatementScenarioAxis_label_en-US" xlink:label="lab_srt_StatementScenarioAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Scenario [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_StatementScenarioAxis" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_StatementScenarioAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_StatementScenarioAxis" xlink:to="lab_srt_StatementScenarioAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween10000000And14999999Member_5cc44fc0-6a4a-4a46-bdc4-b8864ccb63b9_terseLabel_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween10000000And14999999Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $10,000,000 and $14,999,999</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween10000000And14999999Member_label_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween10000000And14999999Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $10,000,000 and $14,999,999 [Member]</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween10000000And14999999Member_documentation_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween10000000And14999999Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $10,000,000 and $14,999,999</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween10000000And14999999Member" xlink:href="efc-20201231.xsd#efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween10000000And14999999Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween10000000And14999999Member" xlink:to="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween10000000And14999999Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AllocatedShareBasedCompensationExpense_fd6180b4-d2a0-4a10-9869-19ab0eeaf2ab_terseLabel_en-US" xlink:label="lab_us-gaap_AllocatedShareBasedCompensationExpense" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based long term incentive plan unit expense</link:label>
    <link:label id="lab_us-gaap_AllocatedShareBasedCompensationExpense_3775ea25-26eb-4c3d-8c9d-111a16fde984_verboseLabel_en-US" xlink:label="lab_us-gaap_AllocatedShareBasedCompensationExpense" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based long term incentive plan unit awards (non-cash)</link:label>
    <link:label id="lab_us-gaap_AllocatedShareBasedCompensationExpense_label_en-US" xlink:label="lab_us-gaap_AllocatedShareBasedCompensationExpense" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Payment Arrangement, Expense</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AllocatedShareBasedCompensationExpense" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AllocatedShareBasedCompensationExpense"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AllocatedShareBasedCompensationExpense" xlink:to="lab_us-gaap_AllocatedShareBasedCompensationExpense" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RelatedPartyTransactionDomain_b0c20bba-d54a-4c1a-8408-6065c42d3fed_terseLabel_en-US" xlink:label="lab_us-gaap_RelatedPartyTransactionDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party Transaction [Domain]</link:label>
    <link:label id="lab_us-gaap_RelatedPartyTransactionDomain_label_en-US" xlink:label="lab_us-gaap_RelatedPartyTransactionDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party Transaction [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RelatedPartyTransactionDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RelatedPartyTransactionDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RelatedPartyTransactionDomain" xlink:to="lab_us-gaap_RelatedPartyTransactionDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2030Member_da4225c1-d007-4166-ad85-8a49590837ea_terseLabel_en-US" xlink:label="lab_efc_A2030Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2030 [Member]</link:label>
    <link:label id="lab_efc_A2030Member_label_en-US" xlink:label="lab_efc_A2030Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2030 [Member]</link:label>
    <link:label id="lab_efc_A2030Member_documentation_en-US" xlink:label="lab_efc_A2030Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2030 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2030Member" xlink:href="efc-20201231.xsd#efc_A2030Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2030Member" xlink:to="lab_efc_A2030Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesAssetsMember_02d505ab-4bf4-4b8f-b069-3e0131c34aed_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesAssetsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities</link:label>
    <link:label id="lab_us-gaap_SecuritiesAssetsMember_label_en-US" xlink:label="lab_us-gaap_SecuritiesAssetsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities (Assets) [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesAssetsMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesAssetsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesAssetsMember" xlink:to="lab_us-gaap_SecuritiesAssetsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityAddressCityOrTown_cd40ab72-eba4-418c-b679-09afcf085bba_terseLabel_en-US" xlink:label="lab_dei_EntityAddressCityOrTown" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Address, City or Town</link:label>
    <link:label id="lab_dei_EntityAddressCityOrTown_label_en-US" xlink:label="lab_dei_EntityAddressCityOrTown" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Address, City or Town</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityAddressCityOrTown" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityAddressCityOrTown"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityAddressCityOrTown" xlink:to="lab_dei_EntityAddressCityOrTown" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue1939U.S.TreasuryNoteCoupon2.75MaturityDate202308Member_f58900d9-431e-4b02-9c21-9e1d04bece7e_terseLabel_en-US" xlink:label="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue1939U.S.TreasuryNoteCoupon2.75MaturityDate202308Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of America Securities Collateralized by Par Value $1,939 U.S. Treasury Note, Coupon 2.75%, Maturity Date 2023 08 [Member]</link:label>
    <link:label id="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue1939U.S.TreasuryNoteCoupon2.75MaturityDate202308Member_label_en-US" xlink:label="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue1939U.S.TreasuryNoteCoupon2.75MaturityDate202308Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of America Securities Collateralized by Par Value $1,939 U.S. Treasury Note, Coupon 2.75%, Maturity Date 2023 08 [Member]</link:label>
    <link:label id="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue1939U.S.TreasuryNoteCoupon2.75MaturityDate202308Member_documentation_en-US" xlink:label="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue1939U.S.TreasuryNoteCoupon2.75MaturityDate202308Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of America Securities Collateralized by Par Value $1,939 U.S. Treasury Note, Coupon 2.75%, Maturity Date 2023 08 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_BankofAmericaSecuritiesCollateralizedbyParValue1939U.S.TreasuryNoteCoupon2.75MaturityDate202308Member" xlink:href="efc-20201231.xsd#efc_BankofAmericaSecuritiesCollateralizedbyParValue1939U.S.TreasuryNoteCoupon2.75MaturityDate202308Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_BankofAmericaSecuritiesCollateralizedbyParValue1939U.S.TreasuryNoteCoupon2.75MaturityDate202308Member" xlink:to="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue1939U.S.TreasuryNoteCoupon2.75MaturityDate202308Member" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_DocumentAnnualReport_d69cefcb-7b07-4fa6-9861-828c348f4927_terseLabel_en-US" xlink:label="lab_dei_DocumentAnnualReport" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Document Annual Report</link:label>
    <link:label id="lab_dei_DocumentAnnualReport_label_en-US" xlink:label="lab_dei_DocumentAnnualReport" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Document Annual Report</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_DocumentAnnualReport" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_DocumentAnnualReport"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_DocumentAnnualReport" xlink:to="lab_dei_DocumentAnnualReport" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Unrealizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet_b1b628e3-6677-4185-9acb-792c216f8dc1_terseLabel_en-US" xlink:label="lab_efc_Unrealizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized gains (losses) on securities and loans, financial derivatives, and real estate owned, net</link:label>
    <link:label id="lab_efc_Unrealizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet_label_en-US" xlink:label="lab_efc_Unrealizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized gains (losses) on securities and loans, financial derivatives, and real estate owned, net</link:label>
    <link:label id="lab_efc_Unrealizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet_documentation_en-US" xlink:label="lab_efc_Unrealizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized gains (losses) on securities and loans, financial derivatives, and real estate owned, net</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Unrealizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet" xlink:href="efc-20201231.xsd#efc_Unrealizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Unrealizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet" xlink:to="lab_efc_Unrealizedgainslossesonsecuritiesandloansfinancialderivativesandrealestateownednet" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfDerivativeInstrumentsGainLossInStatementOfFinancialPerformanceTextBlock_64c3a556-7d76-4967-b815-73c54853c1bf_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfDerivativeInstrumentsGainLossInStatementOfFinancialPerformanceTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Gains and Losses on Derivative Contracts</link:label>
    <link:label id="lab_us-gaap_ScheduleOfDerivativeInstrumentsGainLossInStatementOfFinancialPerformanceTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfDerivativeInstrumentsGainLossInStatementOfFinancialPerformanceTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Instruments, Gain (Loss) [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfDerivativeInstrumentsGainLossInStatementOfFinancialPerformanceTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfDerivativeInstrumentsGainLossInStatementOfFinancialPerformanceTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfDerivativeInstrumentsGainLossInStatementOfFinancialPerformanceTextBlock" xlink:to="lab_us-gaap_ScheduleOfDerivativeInstrumentsGainLossInStatementOfFinancialPerformanceTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesFinancingTransactionAxis_8a216752-1ae0-4b5d-85ab-97622ab71f3f_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesFinancingTransactionAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Financing Transaction [Axis]</link:label>
    <link:label id="lab_us-gaap_SecuritiesFinancingTransactionAxis_label_en-US" xlink:label="lab_us-gaap_SecuritiesFinancingTransactionAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Financing Transaction [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesFinancingTransactionAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesFinancingTransactionAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesFinancingTransactionAxis" xlink:to="lab_us-gaap_SecuritiesFinancingTransactionAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CashAndCashEquivalentsMember_c24e4ceb-3bce-4ba4-ae87-9bb1a144653d_terseLabel_en-US" xlink:label="lab_us-gaap_CashAndCashEquivalentsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash and Cash Equivalents [Member]</link:label>
    <link:label id="lab_us-gaap_CashAndCashEquivalentsMember_label_en-US" xlink:label="lab_us-gaap_CashAndCashEquivalentsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash and Cash Equivalents [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CashAndCashEquivalentsMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CashAndCashEquivalentsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CashAndCashEquivalentsMember" xlink:to="lab_us-gaap_CashAndCashEquivalentsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DividendsPayableCurrentAndNoncurrent_21c43064-5240-4eda-a579-09537c384c78_terseLabel_en-US" xlink:label="lab_us-gaap_DividendsPayableCurrentAndNoncurrent" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends payable</link:label>
    <link:label id="lab_us-gaap_DividendsPayableCurrentAndNoncurrent_label_en-US" xlink:label="lab_us-gaap_DividendsPayableCurrentAndNoncurrent" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends Payable</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DividendsPayableCurrentAndNoncurrent" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DividendsPayableCurrentAndNoncurrent"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DividendsPayableCurrentAndNoncurrent" xlink:to="lab_us-gaap_DividendsPayableCurrentAndNoncurrent" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_IndustrialSectorMember_3d109a17-c41e-4178-86b8-e0dcb2a142be_terseLabel_en-US" xlink:label="lab_efc_IndustrialSectorMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Industrial Sector [Member]</link:label>
    <link:label id="lab_efc_IndustrialSectorMember_label_en-US" xlink:label="lab_efc_IndustrialSectorMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Industrial Sector [Member]</link:label>
    <link:label id="lab_efc_IndustrialSectorMember_documentation_en-US" xlink:label="lab_efc_IndustrialSectorMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Industrial Sector [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_IndustrialSectorMember" xlink:href="efc-20201231.xsd#efc_IndustrialSectorMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_IndustrialSectorMember" xlink:to="lab_efc_IndustrialSectorMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_IncreaseDecreaseInterestAndDividendsPayable_ca4a3a79-404c-4ab7-be69-572f7f779612_terseLabel_en-US" xlink:label="lab_efc_IncreaseDecreaseInterestAndDividendsPayable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest and dividends payable</link:label>
    <link:label id="lab_efc_IncreaseDecreaseInterestAndDividendsPayable_label_en-US" xlink:label="lab_efc_IncreaseDecreaseInterestAndDividendsPayable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase Decrease Interest And Dividends Payable</link:label>
    <link:label id="lab_efc_IncreaseDecreaseInterestAndDividendsPayable_documentation_en-US" xlink:label="lab_efc_IncreaseDecreaseInterestAndDividendsPayable" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (decrease) in the sum of the carrying values as of the balance sheet date of (a) interest payable on all forms of debt, including trade payables, that has been incurred, and (b) dividends declared but unpaid on equity securities issued by the entity and outstanding (also includes dividends collected on behalf of another owner of securities that are being held by the entity).</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_IncreaseDecreaseInterestAndDividendsPayable" xlink:href="efc-20201231.xsd#efc_IncreaseDecreaseInterestAndDividendsPayable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_IncreaseDecreaseInterestAndDividendsPayable" xlink:to="lab_efc_IncreaseDecreaseInterestAndDividendsPayable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_VariableInterestEntitiesByClassificationOfEntityAxis_80a052ef-fd00-46f8-a17b-9fa7d3f9e3df_terseLabel_en-US" xlink:label="lab_us-gaap_VariableInterestEntitiesByClassificationOfEntityAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Variable Interest Entities [Axis]</link:label>
    <link:label id="lab_us-gaap_VariableInterestEntitiesByClassificationOfEntityAxis_label_en-US" xlink:label="lab_us-gaap_VariableInterestEntitiesByClassificationOfEntityAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Variable Interest Entities [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_VariableInterestEntitiesByClassificationOfEntityAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_VariableInterestEntitiesByClassificationOfEntityAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_VariableInterestEntitiesByClassificationOfEntityAxis" xlink:to="lab_us-gaap_VariableInterestEntitiesByClassificationOfEntityAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PreferredStockSharesAuthorized_507e7ed5-553b-459a-bbbe-88dffaa42e14_terseLabel_en-US" xlink:label="lab_us-gaap_PreferredStockSharesAuthorized" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Shares Authorized</link:label>
    <link:label id="lab_us-gaap_PreferredStockSharesAuthorized_label_en-US" xlink:label="lab_us-gaap_PreferredStockSharesAuthorized" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Shares Authorized</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PreferredStockSharesAuthorized" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PreferredStockSharesAuthorized"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PreferredStockSharesAuthorized" xlink:to="lab_us-gaap_PreferredStockSharesAuthorized" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PortionAtFairValueFairValueDisclosureMember_f4cb0a21-b9ee-4717-b97c-14ac66e8e538_terseLabel_en-US" xlink:label="lab_us-gaap_PortionAtFairValueFairValueDisclosureMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Portion at Fair Value Measurement [Member]</link:label>
    <link:label id="lab_us-gaap_PortionAtFairValueFairValueDisclosureMember_label_en-US" xlink:label="lab_us-gaap_PortionAtFairValueFairValueDisclosureMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Portion at Fair Value Measurement [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PortionAtFairValueFairValueDisclosureMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PortionAtFairValueFairValueDisclosureMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PortionAtFairValueFairValueDisclosureMember" xlink:to="lab_us-gaap_PortionAtFairValueFairValueDisclosureMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_UnsecuredDebtMember_e636a69d-f988-4c9e-a9be-1d73dadc5689_terseLabel_en-US" xlink:label="lab_us-gaap_UnsecuredDebtMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Senior Notes</link:label>
    <link:label id="lab_us-gaap_UnsecuredDebtMember_label_en-US" xlink:label="lab_us-gaap_UnsecuredDebtMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unsecured Debt [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_UnsecuredDebtMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_UnsecuredDebtMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_UnsecuredDebtMember" xlink:to="lab_us-gaap_UnsecuredDebtMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureTransferFromMortgageLoansNumberOfProperties_e6dbe5da-023a-4116-bce4-af6458127028_terseLabel_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureTransferFromMortgageLoansNumberOfProperties" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transfers from mortgage loans</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureTransferFromMortgageLoansNumberOfProperties_label_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureTransferFromMortgageLoansNumberOfProperties" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure, Transfer From Mortgage Loans, Number Of Properties</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureTransferFromMortgageLoansNumberOfProperties_documentation_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureTransferFromMortgageLoansNumberOfProperties" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure, Transfer From Mortgage Loans, Number Of Properties</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RealEstateAcquiredThroughForeclosureTransferFromMortgageLoansNumberOfProperties" xlink:href="efc-20201231.xsd#efc_RealEstateAcquiredThroughForeclosureTransferFromMortgageLoansNumberOfProperties"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RealEstateAcquiredThroughForeclosureTransferFromMortgageLoansNumberOfProperties" xlink:to="lab_efc_RealEstateAcquiredThroughForeclosureTransferFromMortgageLoansNumberOfProperties" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporation30Year4Point50PercentMember_672d77b9-3b79-490b-adde-b0ec5979bc81_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporation30Year4Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation (30 Year) 4 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporation30Year4Point50PercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporation30Year4Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation (30 Year) 4 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporation30Year4Point50PercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporation30Year4Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation (30 Year) 4.50%</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporation30Year4Point50PercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporation30Year4Point50PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporation30Year4Point50PercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporation30Year4Point50PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTable_b7911873-5ae1-4edf-86c7-cad370c4624e_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Assets Measured on Recurring Basis, Unobservable Input Reconciliation [Table]</link:label>
    <link:label id="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTable_label_en-US" xlink:label="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Assets Measured on Recurring Basis, Unobservable Input Reconciliation [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTable" xlink:to="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StockholdersEquityAbstract_62a75442-7afa-4b89-8bc5-89371f6e99e1_terseLabel_en-US" xlink:label="lab_us-gaap_StockholdersEquityAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">ANALYSIS OF EQUITY:</link:label>
    <link:label id="lab_us-gaap_StockholdersEquityAbstract_label_en-US" xlink:label="lab_us-gaap_StockholdersEquityAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stockholders' Equity Attributable to Parent [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StockholdersEquityAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockholdersEquityAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StockholdersEquityAbstract" xlink:to="lab_us-gaap_StockholdersEquityAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue2062U.S.TreasuryNoteCoupon2.88MaturityDate202311Member_9745bd99-2221-4c16-9cd3-2366dae0d224_terseLabel_en-US" xlink:label="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue2062U.S.TreasuryNoteCoupon2.88MaturityDate202311Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of America Securities Collateralized by Par Value $2,062 U.S. Treasury Note, Coupon 2.88%, Maturity Date 2023 11 [Member]</link:label>
    <link:label id="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue2062U.S.TreasuryNoteCoupon2.88MaturityDate202311Member_label_en-US" xlink:label="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue2062U.S.TreasuryNoteCoupon2.88MaturityDate202311Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of America Securities Collateralized by Par Value $2,062 U.S. Treasury Note, Coupon 2.88%, Maturity Date 2023 11 [Member]</link:label>
    <link:label id="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue2062U.S.TreasuryNoteCoupon2.88MaturityDate202311Member_documentation_en-US" xlink:label="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue2062U.S.TreasuryNoteCoupon2.88MaturityDate202311Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of America Securities Collateralized by Par Value $2,062 U.S. Treasury Note, Coupon 2.88%, Maturity Date 2023 11 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_BankofAmericaSecuritiesCollateralizedbyParValue2062U.S.TreasuryNoteCoupon2.88MaturityDate202311Member" xlink:href="efc-20201231.xsd#efc_BankofAmericaSecuritiesCollateralizedbyParValue2062U.S.TreasuryNoteCoupon2.88MaturityDate202311Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_BankofAmericaSecuritiesCollateralizedbyParValue2062U.S.TreasuryNoteCoupon2.88MaturityDate202311Member" xlink:to="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue2062U.S.TreasuryNoteCoupon2.88MaturityDate202311Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueByBalanceSheetGroupingTextBlock_541d4ec7-5e64-4827-a97c-67b13d1a891b_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueByBalanceSheetGroupingTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Financial Instruments</link:label>
    <link:label id="lab_us-gaap_FairValueByBalanceSheetGroupingTextBlock_label_en-US" xlink:label="lab_us-gaap_FairValueByBalanceSheetGroupingTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, by Balance Sheet Grouping [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueByBalanceSheetGroupingTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueByBalanceSheetGroupingTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueByBalanceSheetGroupingTextBlock" xlink:to="lab_us-gaap_FairValueByBalanceSheetGroupingTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeLiabilities_2bb3047e-b8f5-477d-b7b0-3b09deb56ab8_verboseLabel_en-US" xlink:label="lab_us-gaap_DerivativeLiabilities" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial derivatives&#8212;liabilities, at fair value</link:label>
    <link:label id="lab_us-gaap_DerivativeLiabilities_fca3bedc-7c26-4d51-9742-e876708ca36b_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeLiabilities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Liability</link:label>
    <link:label id="lab_us-gaap_DerivativeLiabilities_ab16768f-97d0-419f-b380-141aedf81ecc_negatedLabel_en-US" xlink:label="lab_us-gaap_DerivativeLiabilities" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial derivatives&#8211;liabilities, at fair value-</link:label>
    <link:label id="lab_us-gaap_DerivativeLiabilities_label_en-US" xlink:label="lab_us-gaap_DerivativeLiabilities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Liability</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeLiabilities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeLiabilities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeLiabilities" xlink:to="lab_us-gaap_DerivativeLiabilities" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SubsequentEventsTextBlock_476b87a2-03b5-4cfa-ab38-f5d683d65d9b_terseLabel_en-US" xlink:label="lab_us-gaap_SubsequentEventsTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsequent Events</link:label>
    <link:label id="lab_us-gaap_SubsequentEventsTextBlock_label_en-US" xlink:label="lab_us-gaap_SubsequentEventsTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsequent Events [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SubsequentEventsTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SubsequentEventsTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SubsequentEventsTextBlock" xlink:to="lab_us-gaap_SubsequentEventsTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OffsettingOfAssetsAndLiabilitiesTable_5434a159-ca34-4f77-a49a-5c486e33f4c3_terseLabel_en-US" xlink:label="lab_efc_OffsettingOfAssetsAndLiabilitiesTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Offsetting of Assets and Liabilities [Table]</link:label>
    <link:label id="lab_efc_OffsettingOfAssetsAndLiabilitiesTable_label_en-US" xlink:label="lab_efc_OffsettingOfAssetsAndLiabilitiesTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Offsetting of Assets and Liabilities [Table]</link:label>
    <link:label id="lab_efc_OffsettingOfAssetsAndLiabilitiesTable_documentation_en-US" xlink:label="lab_efc_OffsettingOfAssetsAndLiabilitiesTable" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Offsetting of Assets and Liabilities [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OffsettingOfAssetsAndLiabilitiesTable" xlink:href="efc-20201231.xsd#efc_OffsettingOfAssetsAndLiabilitiesTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OffsettingOfAssetsAndLiabilitiesTable" xlink:to="lab_efc_OffsettingOfAssetsAndLiabilitiesTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate_b40854e1-2759-48a1-ae9a-ae3a2845e07c_terseLabel_en-US" xlink:label="lab_us-gaap_EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Effective Income Tax Rate Reconciliation, at Federal Statutory Income Tax Rate, Percent</link:label>
    <link:label id="lab_us-gaap_EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate_label_en-US" xlink:label="lab_us-gaap_EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Effective Income Tax Rate Reconciliation, at Federal Statutory Income Tax Rate, Percent</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate" xlink:to="lab_us-gaap_EffectiveIncomeTaxRateReconciliationAtFederalStatutoryIncomeTaxRate" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationTaxexpensebenefit_c2668e55-bf0e-46f6-8559-d043288da165_terseLabel_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationTaxexpensebenefit" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Tax expense (benefit)</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationTaxexpensebenefit_label_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationTaxexpensebenefit" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Tax expense (benefit)</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationTaxexpensebenefit_documentation_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationTaxexpensebenefit" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount of tax expense (benefit) reported by an equity method investment of the entity.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationTaxexpensebenefit" xlink:href="efc-20201231.xsd#efc_EquityMethodInvestmentSummarizedFinancialInformationTaxexpensebenefit"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationTaxexpensebenefit" xlink:to="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationTaxexpensebenefit" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_IssuedDelayedDeliverySecuritiesPolicyPolicyTextBlock_3da5964b-a7cd-49b8-9cb1-11909b1b0bc9_terseLabel_en-US" xlink:label="lab_efc_IssuedDelayedDeliverySecuritiesPolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">When-Issued/Delayed Delivery Securities</link:label>
    <link:label id="lab_efc_IssuedDelayedDeliverySecuritiesPolicyPolicyTextBlock_label_en-US" xlink:label="lab_efc_IssuedDelayedDeliverySecuritiesPolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Issued Delayed Delivery Securities Policy [Policy Text Block]</link:label>
    <link:label id="lab_efc_IssuedDelayedDeliverySecuritiesPolicyPolicyTextBlock_documentation_en-US" xlink:label="lab_efc_IssuedDelayedDeliverySecuritiesPolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disclosure of accounting policy for forward settling transactions in the To Be Announced MBS market.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_IssuedDelayedDeliverySecuritiesPolicyPolicyTextBlock" xlink:href="efc-20201231.xsd#efc_IssuedDelayedDeliverySecuritiesPolicyPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_IssuedDelayedDeliverySecuritiesPolicyPolicyTextBlock" xlink:to="lab_efc_IssuedDelayedDeliverySecuritiesPolicyPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ConcentrationRiskDisclosureTextBlock_b4ef1ad8-9d1b-42fb-b56c-55f6a1db490a_terseLabel_en-US" xlink:label="lab_us-gaap_ConcentrationRiskDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Counterparty Risk</link:label>
    <link:label id="lab_us-gaap_ConcentrationRiskDisclosureTextBlock_label_en-US" xlink:label="lab_us-gaap_ConcentrationRiskDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk Disclosure [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ConcentrationRiskDisclosureTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ConcentrationRiskDisclosureTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ConcentrationRiskDisclosureTextBlock" xlink:to="lab_us-gaap_ConcentrationRiskDisclosureTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePercentMember_978a08ad-d102-409b-b269-6f994fea4af3_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsLineItems_49cd543a-6c06-4f58-804a-58cde272670e_terseLabel_en-US" xlink:label="lab_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Organization, Consolidation and Presentation of Financial Statements [Line Items]</link:label>
    <link:label id="lab_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsLineItems_label_en-US" xlink:label="lab_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Organization, Consolidation and Presentation of Financial Statements [Line Items]</link:label>
    <link:label id="lab_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsLineItems_documentation_en-US" xlink:label="lab_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsLineItems" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Line Items] for Organization, Consolidation and Presentation of Financial Statements [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsLineItems" xlink:href="efc-20201231.xsd#efc_OrganizationConsolidationAndPresentationOfFinancialStatementsLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsLineItems" xlink:to="lab_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PercentageOfTotalReceivableForSecuritiesSold_c2ed0dc3-77ce-470a-be25-3656540e9b0d_terseLabel_en-US" xlink:label="lab_efc_PercentageOfTotalReceivableForSecuritiesSold" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">%&#160;of&#160;Total&#160;Receivable for Securities Sold</link:label>
    <link:label id="lab_efc_PercentageOfTotalReceivableForSecuritiesSold_label_en-US" xlink:label="lab_efc_PercentageOfTotalReceivableForSecuritiesSold" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage Of Total Receivable For Securities Sold</link:label>
    <link:label id="lab_efc_PercentageOfTotalReceivableForSecuritiesSold_documentation_en-US" xlink:label="lab_efc_PercentageOfTotalReceivableForSecuritiesSold" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage of total receivables for securities sold due from a specific counterparty above a specified concentration risk threshold.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PercentageOfTotalReceivableForSecuritiesSold" xlink:href="efc-20201231.xsd#efc_PercentageOfTotalReceivableForSecuritiesSold"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PercentageOfTotalReceivableForSecuritiesSold" xlink:to="lab_efc_PercentageOfTotalReceivableForSecuritiesSold" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EffectiveIncomeTaxRateReconciliationTaxExemptIncome_e5bb4a84-fc87-45cc-a63f-ee94c0e6b013_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_EffectiveIncomeTaxRateReconciliationTaxExemptIncome" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Effective Income Tax Rate Reconciliation, Tax Exempt Income, Percent</link:label>
    <link:label id="lab_us-gaap_EffectiveIncomeTaxRateReconciliationTaxExemptIncome_label_en-US" xlink:label="lab_us-gaap_EffectiveIncomeTaxRateReconciliationTaxExemptIncome" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Effective Income Tax Rate Reconciliation, Tax Exempt Income, Percent</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EffectiveIncomeTaxRateReconciliationTaxExemptIncome" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EffectiveIncomeTaxRateReconciliationTaxExemptIncome"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EffectiveIncomeTaxRateReconciliationTaxExemptIncome" xlink:to="lab_us-gaap_EffectiveIncomeTaxRateReconciliationTaxExemptIncome" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueMeasurementsRecurringMember_1f1c1375-5250-42c7-981a-c3b782eea691_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueMeasurementsRecurringMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurements, Recurring</link:label>
    <link:label id="lab_us-gaap_FairValueMeasurementsRecurringMember_label_en-US" xlink:label="lab_us-gaap_FairValueMeasurementsRecurringMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Recurring [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueMeasurementsRecurringMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueMeasurementsRecurringMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueMeasurementsRecurringMember" xlink:to="lab_us-gaap_FairValueMeasurementsRecurringMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RealEstateOwnedTextBlock_67b38cde-0e75-4fde-9bcb-c34899542aa8_terseLabel_en-US" xlink:label="lab_us-gaap_RealEstateOwnedTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Owned</link:label>
    <link:label id="lab_us-gaap_RealEstateOwnedTextBlock_label_en-US" xlink:label="lab_us-gaap_RealEstateOwnedTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Owned [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RealEstateOwnedTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RealEstateOwnedTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RealEstateOwnedTextBlock" xlink:to="lab_us-gaap_RealEstateOwnedTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RelatedpartytrustResidentialMortgageLoansandREOMember_88e0aad4-3fac-411e-becb-7576928e2c77_terseLabel_en-US" xlink:label="lab_efc_RelatedpartytrustResidentialMortgageLoansandREOMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related party trust - Residential Mortgage Loans and REO</link:label>
    <link:label id="lab_efc_RelatedpartytrustResidentialMortgageLoansandREOMember_label_en-US" xlink:label="lab_efc_RelatedpartytrustResidentialMortgageLoansandREOMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related party trust - Residential Mortgage Loans and REO [Member]</link:label>
    <link:label id="lab_efc_RelatedpartytrustResidentialMortgageLoansandREOMember_documentation_en-US" xlink:label="lab_efc_RelatedpartytrustResidentialMortgageLoansandREOMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related party trust - Residential Mortgage Loans and REO [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RelatedpartytrustResidentialMortgageLoansandREOMember" xlink:href="efc-20201231.xsd#efc_RelatedpartytrustResidentialMortgageLoansandREOMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RelatedpartytrustResidentialMortgageLoansandREOMember" xlink:to="lab_efc_RelatedpartytrustResidentialMortgageLoansandREOMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseTypeDomain_7faddd99-4a42-4a11-8c70-2bba4e09c040_terseLabel_en-US" xlink:label="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseTypeDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets Sold under Agreements to Repurchase, Type [Domain]</link:label>
    <link:label id="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseTypeDomain_label_en-US" xlink:label="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseTypeDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets Sold under Agreements to Repurchase, Type [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AssetsSoldUnderAgreementsToRepurchaseTypeDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AssetsSoldUnderAgreementsToRepurchaseTypeDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AssetsSoldUnderAgreementsToRepurchaseTypeDomain" xlink:to="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseTypeDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FixedRateAgencySecuritiesMember_c850281f-4b5a-4156-bdb8-61d1872b10e7_terseLabel_en-US" xlink:label="lab_efc_FixedRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed Rate Agency Securities [Member]</link:label>
    <link:label id="lab_efc_FixedRateAgencySecuritiesMember_label_en-US" xlink:label="lab_efc_FixedRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed Rate Agency Securities [Member]</link:label>
    <link:label id="lab_efc_FixedRateAgencySecuritiesMember_documentation_en-US" xlink:label="lab_efc_FixedRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities issued by government agencies or government sponsored entities with a fixed interest rate.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FixedRateAgencySecuritiesMember" xlink:href="efc-20201231.xsd#efc_FixedRateAgencySecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FixedRateAgencySecuritiesMember" xlink:to="lab_efc_FixedRateAgencySecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearThree_7032117c-1857-445f-9126-f53bebac1fc0_terseLabel_en-US" xlink:label="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearThree" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2020 Scheduled Repayment of Principal</link:label>
    <link:label id="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearThree_label_en-US" xlink:label="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearThree" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Maturities, Repayments of Principal in Rolling Year Three</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearThree" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearThree"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearThree" xlink:to="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearThree" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncreaseDecreaseInOperatingAssetsAbstract_2130f096-de08-459d-ac33-10403fd27245_terseLabel_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInOperatingAssetsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">(Increase) decrease in assets:</link:label>
    <link:label id="lab_us-gaap_IncreaseDecreaseInOperatingAssetsAbstract_label_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInOperatingAssetsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (Decrease) in Operating Assets [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncreaseDecreaseInOperatingAssetsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncreaseDecreaseInOperatingAssetsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncreaseDecreaseInOperatingAssetsAbstract" xlink:to="lab_us-gaap_IncreaseDecreaseInOperatingAssetsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProceedsFromDebtNetOfIssuanceCosts_eebbcfec-ccdf-4241-afd7-02d2fb68ba66_terseLabel_en-US" xlink:label="lab_us-gaap_ProceedsFromDebtNetOfIssuanceCosts" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from Debt, Net of Issuance Costs</link:label>
    <link:label id="lab_us-gaap_ProceedsFromDebtNetOfIssuanceCosts_label_en-US" xlink:label="lab_us-gaap_ProceedsFromDebtNetOfIssuanceCosts" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from Debt, Net of Issuance Costs</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProceedsFromDebtNetOfIssuanceCosts" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromDebtNetOfIssuanceCosts"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProceedsFromDebtNetOfIssuanceCosts" xlink:to="lab_us-gaap_ProceedsFromDebtNetOfIssuanceCosts" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DividendsCommonStockCash_815cab15-c78a-4d2b-9f27-7943897378df_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_DividendsCommonStockCash" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends</link:label>
    <link:label id="lab_us-gaap_DividendsCommonStockCash_label_en-US" xlink:label="lab_us-gaap_DividendsCommonStockCash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends, Common Stock, Cash</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DividendsCommonStockCash" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DividendsCommonStockCash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DividendsCommonStockCash" xlink:to="lab_us-gaap_DividendsCommonStockCash" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10451U.S.TreasuryNoteCoupon2.88MaturityDate202310Member_7954eaf6-3e5e-4b77-a8c1-09506cd327fe_terseLabel_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10451U.S.TreasuryNoteCoupon2.88MaturityDate202310Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $10,451 U.S. Treasury Note, Coupon 2.88%, Maturity Date 2023 10 [Member]</link:label>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10451U.S.TreasuryNoteCoupon2.88MaturityDate202310Member_label_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10451U.S.TreasuryNoteCoupon2.88MaturityDate202310Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $10,451 U.S. Treasury Note, Coupon 2.88%, Maturity Date 2023 10 [Member]</link:label>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10451U.S.TreasuryNoteCoupon2.88MaturityDate202310Member_documentation_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10451U.S.TreasuryNoteCoupon2.88MaturityDate202310Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $10,451 U.S. Treasury Note, Coupon 2.88%, Maturity Date 2023 10 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10451U.S.TreasuryNoteCoupon2.88MaturityDate202310Member" xlink:href="efc-20201231.xsd#efc_JPMorganSecuritiesLLCCollateralizedbyParValue10451U.S.TreasuryNoteCoupon2.88MaturityDate202310Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10451U.S.TreasuryNoteCoupon2.88MaturityDate202310Member" xlink:to="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10451U.S.TreasuryNoteCoupon2.88MaturityDate202310Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ConcentrationRiskNumberOfCounterpartiesWithConcentrationRiskExposure_c581fff2-089f-419c-8742-ba20b47d7378_terseLabel_en-US" xlink:label="lab_efc_ConcentrationRiskNumberOfCounterpartiesWithConcentrationRiskExposure" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of Counterparties with Exposure</link:label>
    <link:label id="lab_efc_ConcentrationRiskNumberOfCounterpartiesWithConcentrationRiskExposure_label_en-US" xlink:label="lab_efc_ConcentrationRiskNumberOfCounterpartiesWithConcentrationRiskExposure" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk, Number Of Counterparties With Concentration Risk Exposure</link:label>
    <link:label id="lab_efc_ConcentrationRiskNumberOfCounterpartiesWithConcentrationRiskExposure_documentation_en-US" xlink:label="lab_efc_ConcentrationRiskNumberOfCounterpartiesWithConcentrationRiskExposure" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk, Number Of Counterparties With Concentration Risk Exposure</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ConcentrationRiskNumberOfCounterpartiesWithConcentrationRiskExposure" xlink:href="efc-20201231.xsd#efc_ConcentrationRiskNumberOfCounterpartiesWithConcentrationRiskExposure"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ConcentrationRiskNumberOfCounterpartiesWithConcentrationRiskExposure" xlink:to="lab_efc_ConcentrationRiskNumberOfCounterpartiesWithConcentrationRiskExposure" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationRealizedandunrealizedgainlossoninvestments_624089eb-2f60-4a96-b798-5a414926e7f3_terseLabel_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationRealizedandunrealizedgainlossoninvestments" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Realized and unrealized gain (loss) on investments</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationRealizedandunrealizedgainlossoninvestments_label_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationRealizedandunrealizedgainlossoninvestments" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Realized and unrealized gain (loss) on investments</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationRealizedandunrealizedgainlossoninvestments_documentation_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationRealizedandunrealizedgainlossoninvestments" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount of realized and unrealized gains (losses) on investments (including securities, loans, MSRs, and REO) reported by equity method investee.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationRealizedandunrealizedgainlossoninvestments" xlink:href="efc-20201231.xsd#efc_EquityMethodInvestmentSummarizedFinancialInformationRealizedandunrealizedgainlossoninvestments"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationRealizedandunrealizedgainlossoninvestments" xlink:to="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationRealizedandunrealizedgainlossoninvestments" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ClassOfStockLineItems_f383a245-d4a8-44c4-917e-627091301c2d_terseLabel_en-US" xlink:label="lab_us-gaap_ClassOfStockLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Class of Stock [Line Items]</link:label>
    <link:label id="lab_us-gaap_ClassOfStockLineItems_label_en-US" xlink:label="lab_us-gaap_ClassOfStockLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Class of Stock [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ClassOfStockLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ClassOfStockLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ClassOfStockLineItems" xlink:to="lab_us-gaap_ClassOfStockLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentCompanyIncentiveFeeToAverageNetAssets_34afb048-ecf9-4ccd-af46-800661265365_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_InvestmentCompanyIncentiveFeeToAverageNetAssets" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Incentive Fee to Average Net Assets</link:label>
    <link:label id="lab_us-gaap_InvestmentCompanyIncentiveFeeToAverageNetAssets_label_en-US" xlink:label="lab_us-gaap_InvestmentCompanyIncentiveFeeToAverageNetAssets" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Incentive Fee to Average Net Assets</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentCompanyIncentiveFeeToAverageNetAssets" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentCompanyIncentiveFeeToAverageNetAssets"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentCompanyIncentiveFeeToAverageNetAssets" xlink:to="lab_us-gaap_InvestmentCompanyIncentiveFeeToAverageNetAssets" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfEquityMethodInvestmentsTable_3b4bcac7-0ab5-4270-a17f-02293def1c76_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfEquityMethodInvestmentsTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Equity Method Investments [Table]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfEquityMethodInvestmentsTable_label_en-US" xlink:label="lab_us-gaap_ScheduleOfEquityMethodInvestmentsTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Equity Method Investments [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfEquityMethodInvestmentsTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfEquityMethodInvestmentsTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfEquityMethodInvestmentsTable" xlink:to="lab_us-gaap_ScheduleOfEquityMethodInvestmentsTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SubsequentEventsAbstract_02cac9e2-11ee-4239-9527-9703723479a4_terseLabel_en-US" xlink:label="lab_us-gaap_SubsequentEventsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsequent Events [Abstract]</link:label>
    <link:label id="lab_us-gaap_SubsequentEventsAbstract_label_en-US" xlink:label="lab_us-gaap_SubsequentEventsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsequent Events [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SubsequentEventsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SubsequentEventsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SubsequentEventsAbstract" xlink:to="lab_us-gaap_SubsequentEventsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CreditDefaultSwapsOnAssetBackedSecuritiesMember_2948cfef-44c3-4ebb-bf40-0941aeb49ab3_terseLabel_en-US" xlink:label="lab_efc_CreditDefaultSwapsOnAssetBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit default swaps on asset-backed securities</link:label>
    <link:label id="lab_efc_CreditDefaultSwapsOnAssetBackedSecuritiesMember_label_en-US" xlink:label="lab_efc_CreditDefaultSwapsOnAssetBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swaps On Asset Backed Securities [Member]</link:label>
    <link:label id="lab_efc_CreditDefaultSwapsOnAssetBackedSecuritiesMember_documentation_en-US" xlink:label="lab_efc_CreditDefaultSwapsOnAssetBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">A type of swap transaction used as a credit derivative in which one party pays periodic payments to a counterparty and is guaranteed the promise of a pay-off if a referenced asset backed security defaults.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CreditDefaultSwapsOnAssetBackedSecuritiesMember" xlink:href="efc-20201231.xsd#efc_CreditDefaultSwapsOnAssetBackedSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CreditDefaultSwapsOnAssetBackedSecuritiesMember" xlink:to="lab_efc_CreditDefaultSwapsOnAssetBackedSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RelatedPartyTransactionsDisclosureTextBlock_2c19ba65-8405-45c9-a9a0-f975e9a95e72_terseLabel_en-US" xlink:label="lab_us-gaap_RelatedPartyTransactionsDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party Transactions</link:label>
    <link:label id="lab_us-gaap_RelatedPartyTransactionsDisclosureTextBlock_label_en-US" xlink:label="lab_us-gaap_RelatedPartyTransactionsDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party Transactions Disclosure [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RelatedPartyTransactionsDisclosureTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RelatedPartyTransactionsDisclosureTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RelatedPartyTransactionsDisclosureTextBlock" xlink:to="lab_us-gaap_RelatedPartyTransactionsDisclosureTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueByBalanceSheetGroupingTable_13873402-d268-4256-b0c8-af9c386d8633_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueByBalanceSheetGroupingTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, by Balance Sheet Grouping [Table]</link:label>
    <link:label id="lab_us-gaap_FairValueByBalanceSheetGroupingTable_label_en-US" xlink:label="lab_us-gaap_FairValueByBalanceSheetGroupingTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, by Balance Sheet Grouping [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueByBalanceSheetGroupingTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueByBalanceSheetGroupingTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueByBalanceSheetGroupingTable" xlink:to="lab_us-gaap_FairValueByBalanceSheetGroupingTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PreferredStockDividendsPerShareDeclared_e38c57b9-e3d2-446c-be80-2a94495ca152_terseLabel_en-US" xlink:label="lab_us-gaap_PreferredStockDividendsPerShareDeclared" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Dividends Per Share, Declared</link:label>
    <link:label id="lab_us-gaap_PreferredStockDividendsPerShareDeclared_label_en-US" xlink:label="lab_us-gaap_PreferredStockDividendsPerShareDeclared" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Dividends Per Share, Declared</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PreferredStockDividendsPerShareDeclared" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PreferredStockDividendsPerShareDeclared"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PreferredStockDividendsPerShareDeclared" xlink:to="lab_us-gaap_PreferredStockDividendsPerShareDeclared" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MeasurementInputEquityMultipleMember_471052df-845f-4eb0-9e6a-0c4b1e040b13_terseLabel_en-US" xlink:label="lab_efc_MeasurementInputEquityMultipleMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">measurement input equity multiple</link:label>
    <link:label id="lab_efc_MeasurementInputEquityMultipleMember_label_en-US" xlink:label="lab_efc_MeasurementInputEquityMultipleMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">measurement input equity multiple [Member]</link:label>
    <link:label id="lab_efc_MeasurementInputEquityMultipleMember_documentation_en-US" xlink:label="lab_efc_MeasurementInputEquityMultipleMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">measurement input equity multiple</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MeasurementInputEquityMultipleMember" xlink:href="efc-20201231.xsd#efc_MeasurementInputEquityMultipleMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MeasurementInputEquityMultipleMember" xlink:to="lab_efc_MeasurementInputEquityMultipleMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PerformingFinancingReceivableMember_ab07876a-d0ec-4d77-805c-a7c0395db078_verboseLabel_en-US" xlink:label="lab_us-gaap_PerformingFinancingReceivableMember" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Performing and/or Re-performing</link:label>
    <link:label id="lab_us-gaap_PerformingFinancingReceivableMember_5b7896fd-bdff-421b-842c-36a00af63e41_terseLabel_en-US" xlink:label="lab_us-gaap_PerformingFinancingReceivableMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Performing Financial Instruments [Member]</link:label>
    <link:label id="lab_us-gaap_PerformingFinancingReceivableMember_label_en-US" xlink:label="lab_us-gaap_PerformingFinancingReceivableMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Performing Financial Instruments [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PerformingFinancingReceivableMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PerformingFinancingReceivableMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PerformingFinancingReceivableMember" xlink:to="lab_us-gaap_PerformingFinancingReceivableMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritizationTransactionsLineItems_20c55c02-b826-450a-b953-e034ed6e8fc9_terseLabel_en-US" xlink:label="lab_efc_SecuritizationTransactionsLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization Transactions [Line Items]</link:label>
    <link:label id="lab_efc_SecuritizationTransactionsLineItems_label_en-US" xlink:label="lab_efc_SecuritizationTransactionsLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization Transactions [Line Items]</link:label>
    <link:label id="lab_efc_SecuritizationTransactionsLineItems_documentation_en-US" xlink:label="lab_efc_SecuritizationTransactionsLineItems" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Line Items] for Securitization Transactions [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritizationTransactionsLineItems" xlink:href="efc-20201231.xsd#efc_SecuritizationTransactionsLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritizationTransactionsLineItems" xlink:to="lab_efc_SecuritizationTransactionsLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NetUnrealizedGains_d566dae6-85a1-445d-96d5-080015cf568c_terseLabel_en-US" xlink:label="lab_efc_NetUnrealizedGains" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net unrealized gains</link:label>
    <link:label id="lab_efc_NetUnrealizedGains_label_en-US" xlink:label="lab_efc_NetUnrealizedGains" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net unrealized gains</link:label>
    <link:label id="lab_efc_NetUnrealizedGains_documentation_en-US" xlink:label="lab_efc_NetUnrealizedGains" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net unrealized gains</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NetUnrealizedGains" xlink:href="efc-20201231.xsd#efc_NetUnrealizedGains"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NetUnrealizedGains" xlink:to="lab_efc_NetUnrealizedGains" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EarningsPerShareBasicUndistributed_d2bc7288-0318-4808-86bd-b219e2975a35_terseLabel_en-US" xlink:label="lab_us-gaap_EarningsPerShareBasicUndistributed" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Undistributed (Distributed in excess of) (in usd per share)</link:label>
    <link:label id="lab_us-gaap_EarningsPerShareBasicUndistributed_label_en-US" xlink:label="lab_us-gaap_EarningsPerShareBasicUndistributed" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share, Basic, Undistributed</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EarningsPerShareBasicUndistributed" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EarningsPerShareBasicUndistributed"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EarningsPerShareBasicUndistributed" xlink:to="lab_us-gaap_EarningsPerShareBasicUndistributed" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SignificantAccountingPoliciesTable_7f26f231-e148-48f5-a8a6-17f7df6cb410_terseLabel_en-US" xlink:label="lab_efc_SignificantAccountingPoliciesTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Significant Accounting Policies [Table]</link:label>
    <link:label id="lab_efc_SignificantAccountingPoliciesTable_label_en-US" xlink:label="lab_efc_SignificantAccountingPoliciesTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Significant Accounting Policies [Table]</link:label>
    <link:label id="lab_efc_SignificantAccountingPoliciesTable_documentation_en-US" xlink:label="lab_efc_SignificantAccountingPoliciesTable" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Significant Accounting Policies [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SignificantAccountingPoliciesTable" xlink:href="efc-20201231.xsd#efc_SignificantAccountingPoliciesTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SignificantAccountingPoliciesTable" xlink:to="lab_efc_SignificantAccountingPoliciesTable" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_NorthAmericaMember_9ee47efb-330f-4bb6-9260-d3498bb4b109_terseLabel_en-US" xlink:label="lab_srt_NorthAmericaMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">North America [Member]</link:label>
    <link:label id="lab_srt_NorthAmericaMember_label_en-US" xlink:label="lab_srt_NorthAmericaMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">North America [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_NorthAmericaMember" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_NorthAmericaMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_NorthAmericaMember" xlink:to="lab_srt_NorthAmericaMember" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansOnRealEstateCollectionsOfPrincipal_c95a502d-7fd2-4bbd-81ed-65505acbcefa_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateCollectionsOfPrincipal" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Collections of Principal</link:label>
    <link:label id="lab_srt_MortgageLoansOnRealEstateCollectionsOfPrincipal_label_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateCollectionsOfPrincipal" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Collections of Principal</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansOnRealEstateCollectionsOfPrincipal" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansOnRealEstateCollectionsOfPrincipal"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansOnRealEstateCollectionsOfPrincipal" xlink:to="lab_srt_MortgageLoansOnRealEstateCollectionsOfPrincipal" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ContributionsToInvestmentsInNonConsolidatedEntitiesNonCash_5613ac91-c700-4841-b08d-7b5309795660_terseLabel_en-US" xlink:label="lab_efc_ContributionsToInvestmentsInNonConsolidatedEntitiesNonCash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Contributions to investments in non-consolidated entities (non-cash)</link:label>
    <link:label id="lab_efc_ContributionsToInvestmentsInNonConsolidatedEntitiesNonCash_label_en-US" xlink:label="lab_efc_ContributionsToInvestmentsInNonConsolidatedEntitiesNonCash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Contributions to investments in non-consolidated entities (non-cash)</link:label>
    <link:label id="lab_efc_ContributionsToInvestmentsInNonConsolidatedEntitiesNonCash_documentation_en-US" xlink:label="lab_efc_ContributionsToInvestmentsInNonConsolidatedEntitiesNonCash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Contributions to investments in non-consolidated entities (non-cash)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ContributionsToInvestmentsInNonConsolidatedEntitiesNonCash" xlink:href="efc-20201231.xsd#efc_ContributionsToInvestmentsInNonConsolidatedEntitiesNonCash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ContributionsToInvestmentsInNonConsolidatedEntitiesNonCash" xlink:to="lab_efc_ContributionsToInvestmentsInNonConsolidatedEntitiesNonCash" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CommonStocksByIndustryMember_831033ee-3d9f-4816-acbd-878058ef61b7_terseLabel_en-US" xlink:label="lab_us-gaap_CommonStocksByIndustryMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common stock</link:label>
    <link:label id="lab_us-gaap_CommonStocksByIndustryMember_label_en-US" xlink:label="lab_us-gaap_CommonStocksByIndustryMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common Stocks, by Industry [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CommonStocksByIndustryMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CommonStocksByIndustryMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CommonStocksByIndustryMember" xlink:to="lab_us-gaap_CommonStocksByIndustryMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfCapitalizationEquityLineItems_af849810-fb39-4c54-abbe-5f72fc3eb282_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfCapitalizationEquityLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Capitalization, Equity [Line Items]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfCapitalizationEquityLineItems_label_en-US" xlink:label="lab_us-gaap_ScheduleOfCapitalizationEquityLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Capitalization, Equity [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfCapitalizationEquityLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfCapitalizationEquityLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfCapitalizationEquityLineItems" xlink:to="lab_us-gaap_ScheduleOfCapitalizationEquityLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DeferredTaxLiabilitiesInvestments_b6d18d1b-3d7e-4ae9-8e0c-da30138526e8_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_DeferredTaxLiabilitiesInvestments" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Tax Liabilities, Investments</link:label>
    <link:label id="lab_us-gaap_DeferredTaxLiabilitiesInvestments_label_en-US" xlink:label="lab_us-gaap_DeferredTaxLiabilitiesInvestments" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Tax Liabilities, Investments</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DeferredTaxLiabilitiesInvestments" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DeferredTaxLiabilitiesInvestments"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DeferredTaxLiabilitiesInvestments" xlink:to="lab_us-gaap_DeferredTaxLiabilitiesInvestments" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleofOptionsContractsTableTextBlock_f94e3089-ff55-4ca9-b09d-b35c6cba39f5_terseLabel_en-US" xlink:label="lab_efc_ScheduleofOptionsContractsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Options Contracts [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleofOptionsContractsTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleofOptionsContractsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Options Contracts [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleofOptionsContractsTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleofOptionsContractsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Options Contracts [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleofOptionsContractsTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleofOptionsContractsTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleofOptionsContractsTableTextBlock" xlink:to="lab_efc_ScheduleofOptionsContractsTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_UndistributedEarningsAbstract_23f12380-8612-4d9c-82be-462ceb1dc8bb_terseLabel_en-US" xlink:label="lab_us-gaap_UndistributedEarningsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Undistributed (Distributed in excess of) earnings:</link:label>
    <link:label id="lab_us-gaap_UndistributedEarningsAbstract_label_en-US" xlink:label="lab_us-gaap_UndistributedEarningsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Undistributed Earnings, Basic [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_UndistributedEarningsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_UndistributedEarningsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_UndistributedEarningsAbstract" xlink:to="lab_us-gaap_UndistributedEarningsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AmortizationOfFinancingCosts_35e2ce51-819f-4f40-818f-46bdbf6ab91b_terseLabel_en-US" xlink:label="lab_us-gaap_AmortizationOfFinancingCosts" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amortization of deferred debt issuance costs</link:label>
    <link:label id="lab_us-gaap_AmortizationOfFinancingCosts_label_en-US" xlink:label="lab_us-gaap_AmortizationOfFinancingCosts" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amortization of Debt Issuance Costs</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AmortizationOfFinancingCosts" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AmortizationOfFinancingCosts"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AmortizationOfFinancingCosts" xlink:to="lab_us-gaap_AmortizationOfFinancingCosts" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SubsequentEventLineItems_d0314de8-a447-4bbe-9361-6f7098260a1f_terseLabel_en-US" xlink:label="lab_us-gaap_SubsequentEventLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsequent Event [Line Items]</link:label>
    <link:label id="lab_us-gaap_SubsequentEventLineItems_label_en-US" xlink:label="lab_us-gaap_SubsequentEventLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsequent Event [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SubsequentEventLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SubsequentEventLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SubsequentEventLineItems" xlink:to="lab_us-gaap_SubsequentEventLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Aggregateunpaidprincipalbalanceofloanssold_58911756-95c2-49c8-b876-240a7aa3bf77_terseLabel_en-US" xlink:label="lab_efc_Aggregateunpaidprincipalbalanceofloanssold" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Aggregate unpaid principal balance of loans sold</link:label>
    <link:label id="lab_efc_Aggregateunpaidprincipalbalanceofloanssold_label_en-US" xlink:label="lab_efc_Aggregateunpaidprincipalbalanceofloanssold" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Aggregate unpaid principal balance of loans sold</link:label>
    <link:label id="lab_efc_Aggregateunpaidprincipalbalanceofloanssold_documentation_en-US" xlink:label="lab_efc_Aggregateunpaidprincipalbalanceofloanssold" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Aggregate unpaid principal balance of consumer loans sold by the Company and another entity managed by an affiliate of Ellington (the "co-participant").</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Aggregateunpaidprincipalbalanceofloanssold" xlink:href="efc-20201231.xsd#efc_Aggregateunpaidprincipalbalanceofloanssold"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Aggregateunpaidprincipalbalanceofloanssold" xlink:to="lab_efc_Aggregateunpaidprincipalbalanceofloanssold" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodDomain_c4b7275a-5b76-461f-8b6e-613bbc00d776_terseLabel_en-US" xlink:label="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase Agreements and Similar Transactions, Maturity Periods [Domain]</link:label>
    <link:label id="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodDomain_label_en-US" xlink:label="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase Agreements and Similar Transactions, Maturity Periods [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodDomain" xlink:to="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InvestmentsInDebtAndEquitySecuritiesIncludingSecuritiesSoldButNotYetPurchasedTextBlock_522cfbbc-e40c-43d1-9a55-71f48ccb5fbd_terseLabel_en-US" xlink:label="lab_efc_InvestmentsInDebtAndEquitySecuritiesIncludingSecuritiesSoldButNotYetPurchasedTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment in Securities</link:label>
    <link:label id="lab_efc_InvestmentsInDebtAndEquitySecuritiesIncludingSecuritiesSoldButNotYetPurchasedTextBlock_label_en-US" xlink:label="lab_efc_InvestmentsInDebtAndEquitySecuritiesIncludingSecuritiesSoldButNotYetPurchasedTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments In Debt And Equity Securities Including Securities Sold But Not Yet Purchased [Text Block]</link:label>
    <link:label id="lab_efc_InvestmentsInDebtAndEquitySecuritiesIncludingSecuritiesSoldButNotYetPurchasedTextBlock_documentation_en-US" xlink:label="lab_efc_InvestmentsInDebtAndEquitySecuritiesIncludingSecuritiesSoldButNotYetPurchasedTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments In Debt And Equity Securities Including Securities Sold But Not Yet Purchased [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InvestmentsInDebtAndEquitySecuritiesIncludingSecuritiesSoldButNotYetPurchasedTextBlock" xlink:href="efc-20201231.xsd#efc_InvestmentsInDebtAndEquitySecuritiesIncludingSecuritiesSoldButNotYetPurchasedTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InvestmentsInDebtAndEquitySecuritiesIncludingSecuritiesSoldButNotYetPurchasedTextBlock" xlink:to="lab_efc_InvestmentsInDebtAndEquitySecuritiesIncludingSecuritiesSoldButNotYetPurchasedTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Percentagecollaterallizedbygnrcmocertificates_ff63d236-efaf-434f-885f-8889b8b7c7a1_terseLabel_en-US" xlink:label="lab_efc_Percentagecollaterallizedbygnrcmocertificates" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">PercentageCollaterallizedByGNRCMOCertificates</link:label>
    <link:label id="lab_efc_Percentagecollaterallizedbygnrcmocertificates_label_en-US" xlink:label="lab_efc_Percentagecollaterallizedbygnrcmocertificates" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">PercentageCollaterallizedByGNRCMOCertificates</link:label>
    <link:label id="lab_efc_Percentagecollaterallizedbygnrcmocertificates_documentation_en-US" xlink:label="lab_efc_Percentagecollaterallizedbygnrcmocertificates" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage of which private trust is backed by interest in Government National Mortgage Association collateralized mortgage obligation certificates.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Percentagecollaterallizedbygnrcmocertificates" xlink:href="efc-20201231.xsd#efc_Percentagecollaterallizedbygnrcmocertificates"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Percentagecollaterallizedbygnrcmocertificates" xlink:to="lab_efc_Percentagecollaterallizedbygnrcmocertificates" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year5Point00PercentMember_2bd61de7-f939-4d38-8774-cce4f2d9252c_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year5Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 5 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year5Point00PercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year5Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 5 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year5Point00PercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year5Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 5 Point 00 Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociation30Year5Point00PercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociation30Year5Point00PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociation30Year5Point00PercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociation30Year5Point00PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsOtherFourPercentMember_e6bff2d8-39b9-4739-a58f-6d7664424205_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsOtherFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Other) Four Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsOtherFourPercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsOtherFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Other) Four Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsOtherFourPercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsOtherFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Other) Four Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsOtherFourPercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsOtherFourPercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsOtherFourPercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsOtherFourPercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfInterestRateDerivativesTableTextBlock_f21179b0-f4f9-4c35-b0f3-96a21bd7c2f8_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfInterestRateDerivativesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Interest Rate Derivatives [Table Text Block]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfInterestRateDerivativesTableTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfInterestRateDerivativesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Interest Rate Derivatives [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfInterestRateDerivativesTableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfInterestRateDerivativesTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfInterestRateDerivativesTableTextBlock" xlink:to="lab_us-gaap_ScheduleOfInterestRateDerivativesTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_VestDecember312020Member_f2ebea58-6584-4d8a-b780-444d0c9a946f_terseLabel_en-US" xlink:label="lab_efc_VestDecember312020Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 31, 2020 [Member]</link:label>
    <link:label id="lab_efc_VestDecember312020Member_label_en-US" xlink:label="lab_efc_VestDecember312020Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 31, 2020 [Member]</link:label>
    <link:label id="lab_efc_VestDecember312020Member_documentation_en-US" xlink:label="lab_efc_VestDecember312020Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 31, 2020 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_VestDecember312020Member" xlink:href="efc-20201231.xsd#efc_VestDecember312020Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_VestDecember312020Member" xlink:to="lab_efc_VestDecember312020Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LiabilitiesAbstract_76d6c5f0-4c48-42c3-bef5-7406b7c1f6a4_terseLabel_en-US" xlink:label="lab_us-gaap_LiabilitiesAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities</link:label>
    <link:label id="lab_us-gaap_LiabilitiesAbstract_label_en-US" xlink:label="lab_us-gaap_LiabilitiesAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LiabilitiesAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LiabilitiesAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LiabilitiesAbstract" xlink:to="lab_us-gaap_LiabilitiesAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NumberCounterpartiesWithConcentrationRiskReverseRepurchaseAgreements_31b2252f-5950-42d8-b546-69c6168b2a15_terseLabel_en-US" xlink:label="lab_efc_NumberCounterpartiesWithConcentrationRiskReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of Counterparties With Concentration Risk, Reverse Repurchase Agreements</link:label>
    <link:label id="lab_efc_NumberCounterpartiesWithConcentrationRiskReverseRepurchaseAgreements_label_en-US" xlink:label="lab_efc_NumberCounterpartiesWithConcentrationRiskReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number Counterparties With Concentration Risk, Reverse Repurchase Agreements</link:label>
    <link:label id="lab_efc_NumberCounterpartiesWithConcentrationRiskReverseRepurchaseAgreements_documentation_en-US" xlink:label="lab_efc_NumberCounterpartiesWithConcentrationRiskReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The number of counterparites that hold a percentage of open reverse repurchase agreements above a specified concentration risk threshold.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NumberCounterpartiesWithConcentrationRiskReverseRepurchaseAgreements" xlink:href="efc-20201231.xsd#efc_NumberCounterpartiesWithConcentrationRiskReverseRepurchaseAgreements"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NumberCounterpartiesWithConcentrationRiskReverseRepurchaseAgreements" xlink:to="lab_efc_NumberCounterpartiesWithConcentrationRiskReverseRepurchaseAgreements" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_IncreaseDecreaseInPayableForSecuritiesPurchased_24b59444-7fcc-49aa-b47f-845da5d1e41c_terseLabel_en-US" xlink:label="lab_efc_IncreaseDecreaseInPayableForSecuritiesPurchased" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payable for securities purchased and financial derivatives</link:label>
    <link:label id="lab_efc_IncreaseDecreaseInPayableForSecuritiesPurchased_label_en-US" xlink:label="lab_efc_IncreaseDecreaseInPayableForSecuritiesPurchased" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase Decrease In Payable For Securities Purchased</link:label>
    <link:label id="lab_efc_IncreaseDecreaseInPayableForSecuritiesPurchased_documentation_en-US" xlink:label="lab_efc_IncreaseDecreaseInPayableForSecuritiesPurchased" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The increase (decrease) during the reporting period in the aggregate amount of obligations incurred arising from unsettled security purchases.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_IncreaseDecreaseInPayableForSecuritiesPurchased" xlink:href="efc-20201231.xsd#efc_IncreaseDecreaseInPayableForSecuritiesPurchased"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_IncreaseDecreaseInPayableForSecuritiesPurchased" xlink:to="lab_efc_IncreaseDecreaseInPayableForSecuritiesPurchased" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Totalnetrealizedgainloss_cc08a214-a18a-481b-bfdc-9651b223e4f5_totalLabel_en-US" xlink:label="lab_efc_Totalnetrealizedgainloss" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total net realized gain (loss)</link:label>
    <link:label id="lab_efc_Totalnetrealizedgainloss_31cab85e-b73e-44f3-afde-9a5e36ab3a87_terseLabel_en-US" xlink:label="lab_efc_Totalnetrealizedgainloss" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total net realized gain (loss)</link:label>
    <link:label id="lab_efc_Totalnetrealizedgainloss_label_en-US" xlink:label="lab_efc_Totalnetrealizedgainloss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total net realized gain (loss)</link:label>
    <link:label id="lab_efc_Totalnetrealizedgainloss_documentation_en-US" xlink:label="lab_efc_Totalnetrealizedgainloss" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount of realized gain (loss) on investments, financial derivatives, and foreign currency transactions.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Totalnetrealizedgainloss" xlink:href="efc-20201231.xsd#efc_Totalnetrealizedgainloss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Totalnetrealizedgainloss" xlink:to="lab_efc_Totalnetrealizedgainloss" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Proceedsreceivedfromequityinvestmentinriskretentionvehicleliquidation_c47ec0c3-82bd-4633-a557-a81fc80ee60b_terseLabel_en-US" xlink:label="lab_efc_Proceedsreceivedfromequityinvestmentinriskretentionvehicleliquidation" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds received from equity investment in risk retention vehicle (liquidation)</link:label>
    <link:label id="lab_efc_Proceedsreceivedfromequityinvestmentinriskretentionvehicleliquidation_label_en-US" xlink:label="lab_efc_Proceedsreceivedfromequityinvestmentinriskretentionvehicleliquidation" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds received from equity investment in risk retention vehicle (liquidation)</link:label>
    <link:label id="lab_efc_Proceedsreceivedfromequityinvestmentinriskretentionvehicleliquidation_documentation_en-US" xlink:label="lab_efc_Proceedsreceivedfromequityinvestmentinriskretentionvehicleliquidation" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds received in connection with the liquidation of a risk retention vehicle in which the Company held an equity investment. Liquidation of the risk retention vehicle commenced after the CLO Manager determined the CLO Securitization was exempt from the risk retention rules.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Proceedsreceivedfromequityinvestmentinriskretentionvehicleliquidation" xlink:href="efc-20201231.xsd#efc_Proceedsreceivedfromequityinvestmentinriskretentionvehicleliquidation"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Proceedsreceivedfromequityinvestmentinriskretentionvehicleliquidation" xlink:to="lab_efc_Proceedsreceivedfromequityinvestmentinriskretentionvehicleliquidation" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GreaterthansevenyearslessthanelevenMember_b8edf896-3386-4cf6-aeea-742c54362ac2_terseLabel_en-US" xlink:label="lab_efc_GreaterthansevenyearslessthanelevenMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Greater than seven years less than eleven [Member]</link:label>
    <link:label id="lab_efc_GreaterthansevenyearslessthanelevenMember_label_en-US" xlink:label="lab_efc_GreaterthansevenyearslessthanelevenMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Greater than seven years less than eleven [Member]</link:label>
    <link:label id="lab_efc_GreaterthansevenyearslessthanelevenMember_documentation_en-US" xlink:label="lab_efc_GreaterthansevenyearslessthanelevenMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Greater than seven years less than eleven [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GreaterthansevenyearslessthanelevenMember" xlink:href="efc-20201231.xsd#efc_GreaterthansevenyearslessthanelevenMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GreaterthansevenyearslessthanelevenMember" xlink:to="lab_efc_GreaterthansevenyearslessthanelevenMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonQMloanstransferredintosecuritizationtrust_6ac6c977-9dde-41ea-955c-0e6cc2deac23_terseLabel_en-US" xlink:label="lab_efc_NonQMloanstransferredintosecuritizationtrust" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal Balance of Loans Transferred to the Depositor</link:label>
    <link:label id="lab_efc_NonQMloanstransferredintosecuritizationtrust_label_en-US" xlink:label="lab_efc_NonQMloanstransferredintosecuritizationtrust" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-QM loans transferred into securitization trust</link:label>
    <link:label id="lab_efc_NonQMloanstransferredintosecuritizationtrust_documentation_en-US" xlink:label="lab_efc_NonQMloanstransferredintosecuritizationtrust" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair value of non-QM loans transferred into securitization trust.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonQMloanstransferredintosecuritizationtrust" xlink:href="efc-20201231.xsd#efc_NonQMloanstransferredintosecuritizationtrust"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonQMloanstransferredintosecuritizationtrust" xlink:to="lab_efc_NonQMloanstransferredintosecuritizationtrust" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PaymentsOfDividends_2c9452f0-1441-428e-9a54-d52073459eab_negatedLabel_en-US" xlink:label="lab_us-gaap_PaymentsOfDividends" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends paid</link:label>
    <link:label id="lab_us-gaap_PaymentsOfDividends_label_en-US" xlink:label="lab_us-gaap_PaymentsOfDividends" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments of Dividends</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PaymentsOfDividends" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsOfDividends"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PaymentsOfDividends" xlink:to="lab_us-gaap_PaymentsOfDividends" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NumberOfContractsPerEvery1000000InNotionalValue_78fcdcc7-1683-4720-aaef-2676c63d8829_terseLabel_en-US" xlink:label="lab_efc_NumberOfContractsPerEvery1000000InNotionalValue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number Of Contracts Per Every $1,000,000 In Notional Value</link:label>
    <link:label id="lab_efc_NumberOfContractsPerEvery1000000InNotionalValue_label_en-US" xlink:label="lab_efc_NumberOfContractsPerEvery1000000InNotionalValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number Of Contracts Per Every $1,000,000 In Notional Value</link:label>
    <link:label id="lab_efc_NumberOfContractsPerEvery1000000InNotionalValue_documentation_en-US" xlink:label="lab_efc_NumberOfContractsPerEvery1000000InNotionalValue" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number Of Contracts Per Every $1,000,000 In Notional Value</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NumberOfContractsPerEvery1000000InNotionalValue" xlink:href="efc-20201231.xsd#efc_NumberOfContractsPerEvery1000000InNotionalValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NumberOfContractsPerEvery1000000InNotionalValue" xlink:to="lab_efc_NumberOfContractsPerEvery1000000InNotionalValue" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PaymentsToAcquireAvailableForSaleSecuritiesDebt_e8fac38c-c1f8-4aa8-8648-c9e1f3178173_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_PaymentsToAcquireAvailableForSaleSecuritiesDebt" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchase of securities</link:label>
    <link:label id="lab_us-gaap_PaymentsToAcquireAvailableForSaleSecuritiesDebt_label_en-US" xlink:label="lab_us-gaap_PaymentsToAcquireAvailableForSaleSecuritiesDebt" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments to Acquire Debt Securities, Available-for-sale</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PaymentsToAcquireAvailableForSaleSecuritiesDebt" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsToAcquireAvailableForSaleSecuritiesDebt"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PaymentsToAcquireAvailableForSaleSecuritiesDebt" xlink:to="lab_us-gaap_PaymentsToAcquireAvailableForSaleSecuritiesDebt" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageRelatedEntitiesMember_539d310f-5fdf-452b-b372-e76c9d981172_terseLabel_en-US" xlink:label="lab_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageRelatedEntitiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-exchange traded preferred equity investment in mortgage-related entity</link:label>
    <link:label id="lab_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageRelatedEntitiesMember_label_en-US" xlink:label="lab_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageRelatedEntitiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Preferred Equity Investment in Mortgage-Related Entities [Member]</link:label>
    <link:label id="lab_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageRelatedEntitiesMember_documentation_en-US" xlink:label="lab_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageRelatedEntitiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Preferred Equity Investment in Mortgage-Related Entities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageRelatedEntitiesMember" xlink:href="efc-20201231.xsd#efc_NonExchangeTradedPreferredEquityInvestmentinMortgageRelatedEntitiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageRelatedEntitiesMember" xlink:to="lab_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageRelatedEntitiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_VariableInterestEntityPrimaryBeneficiaryMember_ebecbe5b-6e94-41f3-aff5-2f2314d6ce25_terseLabel_en-US" xlink:label="lab_us-gaap_VariableInterestEntityPrimaryBeneficiaryMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Primary Beneficiary</link:label>
    <link:label id="lab_us-gaap_VariableInterestEntityPrimaryBeneficiaryMember_label_en-US" xlink:label="lab_us-gaap_VariableInterestEntityPrimaryBeneficiaryMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Variable Interest Entity, Primary Beneficiary [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_VariableInterestEntityPrimaryBeneficiaryMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_VariableInterestEntityPrimaryBeneficiaryMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_VariableInterestEntityPrimaryBeneficiaryMember" xlink:to="lab_us-gaap_VariableInterestEntityPrimaryBeneficiaryMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonperformingResidentialMortgageLoansandResidentialRealEstateOwnedMember_b13be5b5-8ce6-4d56-9ff6-0862d4136116_terseLabel_en-US" xlink:label="lab_efc_NonperformingResidentialMortgageLoansandResidentialRealEstateOwnedMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-performing residential mortgage loans and residential real estate owned</link:label>
    <link:label id="lab_efc_NonperformingResidentialMortgageLoansandResidentialRealEstateOwnedMember_label_en-US" xlink:label="lab_efc_NonperformingResidentialMortgageLoansandResidentialRealEstateOwnedMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-performing Residential Mortgage Loans and Residential Real Estate Owned [Member]</link:label>
    <link:label id="lab_efc_NonperformingResidentialMortgageLoansandResidentialRealEstateOwnedMember_documentation_en-US" xlink:label="lab_efc_NonperformingResidentialMortgageLoansandResidentialRealEstateOwnedMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-performing Residential Mortgage Loans and Residential Real Estate Owned Properties [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonperformingResidentialMortgageLoansandResidentialRealEstateOwnedMember" xlink:href="efc-20201231.xsd#efc_NonperformingResidentialMortgageLoansandResidentialRealEstateOwnedMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonperformingResidentialMortgageLoansandResidentialRealEstateOwnedMember" xlink:to="lab_efc_NonperformingResidentialMortgageLoansandResidentialRealEstateOwnedMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InterestRatesEquityMarketRiskMember_3aeb156c-6537-4bd9-9eee-917a746e8dce_terseLabel_en-US" xlink:label="lab_efc_InterestRatesEquityMarketRiskMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Rates/Equity Market Risk [Member]</link:label>
    <link:label id="lab_efc_InterestRatesEquityMarketRiskMember_label_en-US" xlink:label="lab_efc_InterestRatesEquityMarketRiskMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Rates/Equity Market Risk [Member]</link:label>
    <link:label id="lab_efc_InterestRatesEquityMarketRiskMember_documentation_en-US" xlink:label="lab_efc_InterestRatesEquityMarketRiskMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Rates/Equity Market Risk [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InterestRatesEquityMarketRiskMember" xlink:href="efc-20201231.xsd#efc_InterestRatesEquityMarketRiskMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InterestRatesEquityMarketRiskMember" xlink:to="lab_efc_InterestRatesEquityMarketRiskMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureAdjustmenttorecordatlowerofcostorfairvalue_c95e5ae0-b6d2-46bc-a077-61a0b2394299_negatedTerseLabel_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureAdjustmenttorecordatlowerofcostorfairvalue" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Adjustments to record at the lower of cost or fair value</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureAdjustmenttorecordatlowerofcostorfairvalue_label_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureAdjustmenttorecordatlowerofcostorfairvalue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure, Adjustment to record at lower of cost or fair value</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureAdjustmenttorecordatlowerofcostorfairvalue_documentation_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureAdjustmenttorecordatlowerofcostorfairvalue" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure, Adjustment to record at lower of cost or fair value</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RealEstateAcquiredThroughForeclosureAdjustmenttorecordatlowerofcostorfairvalue" xlink:href="efc-20201231.xsd#efc_RealEstateAcquiredThroughForeclosureAdjustmenttorecordatlowerofcostorfairvalue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RealEstateAcquiredThroughForeclosureAdjustmenttorecordatlowerofcostorfairvalue" xlink:to="lab_efc_RealEstateAcquiredThroughForeclosureAdjustmenttorecordatlowerofcostorfairvalue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2023Member_2aac0d04-c821-4d85-83af-6276f7975a67_terseLabel_en-US" xlink:label="lab_efc_A2023Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2023 [Member]</link:label>
    <link:label id="lab_efc_A2023Member_label_en-US" xlink:label="lab_efc_A2023Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2023 [Member]</link:label>
    <link:label id="lab_efc_A2023Member_documentation_en-US" xlink:label="lab_efc_A2023Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2023 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2023Member" xlink:href="efc-20201231.xsd#efc_A2023Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2023Member" xlink:to="lab_efc_A2023Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NetCashProvidedByUsedInInvestingActivitiesAbstract_06d2e7c0-e4e9-4138-bf42-4d4e89de53e2_terseLabel_en-US" xlink:label="lab_us-gaap_NetCashProvidedByUsedInInvestingActivitiesAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash Flows from Investing Activities:</link:label>
    <link:label id="lab_us-gaap_NetCashProvidedByUsedInInvestingActivitiesAbstract_label_en-US" xlink:label="lab_us-gaap_NetCashProvidedByUsedInInvestingActivitiesAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Cash Provided by (Used in) Investing Activities [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NetCashProvidedByUsedInInvestingActivitiesAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NetCashProvidedByUsedInInvestingActivitiesAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NetCashProvidedByUsedInInvestingActivitiesAbstract" xlink:to="lab_us-gaap_NetCashProvidedByUsedInInvestingActivitiesAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesFinancingTransactionTypeDomain_1818e9b2-c9e1-4922-9be1-3081286d95f1_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesFinancingTransactionTypeDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Financing Transaction [Domain]</link:label>
    <link:label id="lab_us-gaap_SecuritiesFinancingTransactionTypeDomain_a76d2be7-a1c9-4ed3-8f97-336067856129_verboseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesFinancingTransactionTypeDomain" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization Transactions [Domain]</link:label>
    <link:label id="lab_us-gaap_SecuritiesFinancingTransactionTypeDomain_label_en-US" xlink:label="lab_us-gaap_SecuritiesFinancingTransactionTypeDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Financing Transaction [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesFinancingTransactionTypeDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesFinancingTransactionTypeDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesFinancingTransactionTypeDomain" xlink:to="lab_us-gaap_SecuritiesFinancingTransactionTypeDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MortgageLoansMember_d2052631-09bd-4b0f-bf27-2ffb17d845cf_terseLabel_en-US" xlink:label="lab_efc_MortgageLoansMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage Loans [Member]</link:label>
    <link:label id="lab_efc_MortgageLoansMember_label_en-US" xlink:label="lab_efc_MortgageLoansMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage Loans [Member]</link:label>
    <link:label id="lab_efc_MortgageLoansMember_documentation_en-US" xlink:label="lab_efc_MortgageLoansMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage Loans [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MortgageLoansMember" xlink:href="efc-20201231.xsd#efc_MortgageLoansMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MortgageLoansMember" xlink:to="lab_efc_MortgageLoansMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePercentMember_b13095a5-4fc2-4e26-9c20-effcf8f84cf3_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Three Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Three Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Three Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlementsAndPurchases_7ffbd36c-c410-4f7e-a78a-b4d568ea4859_terseLabel_en-US" xlink:label="lab_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlementsAndPurchases" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities Purchases/Payments</link:label>
    <link:label id="lab_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlementsAndPurchases_label_en-US" xlink:label="lab_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlementsAndPurchases" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurment with Unobservable Inputs Reconciliation, Recurring Basis, Liability, Settlements and Purchases</link:label>
    <link:label id="lab_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlementsAndPurchases_documentation_en-US" xlink:label="lab_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlementsAndPurchases" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Settlements and purchases that have taken place during the period in relation to liabilities measured at fair value and categorized within Level 3 of the fair value hierarchy.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlementsAndPurchases" xlink:href="efc-20201231.xsd#efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlementsAndPurchases"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlementsAndPurchases" xlink:to="lab_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlementsAndPurchases" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InvestmentCreditRatingDomain_9af5ea0a-86f2-4291-a1b6-6e886ca70ade_terseLabel_en-US" xlink:label="lab_efc_InvestmentCreditRatingDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment credit rating [Domain]</link:label>
    <link:label id="lab_efc_InvestmentCreditRatingDomain_label_en-US" xlink:label="lab_efc_InvestmentCreditRatingDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment credit rating [Domain]</link:label>
    <link:label id="lab_efc_InvestmentCreditRatingDomain_documentation_en-US" xlink:label="lab_efc_InvestmentCreditRatingDomain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment credit rating</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InvestmentCreditRatingDomain" xlink:href="efc-20201231.xsd#efc_InvestmentCreditRatingDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InvestmentCreditRatingDomain" xlink:to="lab_efc_InvestmentCreditRatingDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MeasurementInputTypeDomain_b41a4e29-d97f-4d4d-a8ef-d9424095781f_terseLabel_en-US" xlink:label="lab_us-gaap_MeasurementInputTypeDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input Type [Domain]</link:label>
    <link:label id="lab_us-gaap_MeasurementInputTypeDomain_label_en-US" xlink:label="lab_us-gaap_MeasurementInputTypeDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input Type [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MeasurementInputTypeDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MeasurementInputTypeDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MeasurementInputTypeDomain" xlink:to="lab_us-gaap_MeasurementInputTypeDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ProceedsFromInvestmentsSoldShort_a0b7a981-c230-4c77-97b5-c848affddfcc_terseLabel_en-US" xlink:label="lab_efc_ProceedsFromInvestmentsSoldShort" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from investments sold short</link:label>
    <link:label id="lab_efc_ProceedsFromInvestmentsSoldShort_label_en-US" xlink:label="lab_efc_ProceedsFromInvestmentsSoldShort" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds From Investments Sold Short</link:label>
    <link:label id="lab_efc_ProceedsFromInvestmentsSoldShort_documentation_en-US" xlink:label="lab_efc_ProceedsFromInvestmentsSoldShort" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The amount of cash inflow from proceeds received from securities sold short.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ProceedsFromInvestmentsSoldShort" xlink:href="efc-20201231.xsd#efc_ProceedsFromInvestmentsSoldShort"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ProceedsFromInvestmentsSoldShort" xlink:to="lab_efc_ProceedsFromInvestmentsSoldShort" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RelatedPartyDomain_1beea291-3479-4709-a5eb-9fb4e7032be3_terseLabel_en-US" xlink:label="lab_us-gaap_RelatedPartyDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party [Domain]</link:label>
    <link:label id="lab_us-gaap_RelatedPartyDomain_label_en-US" xlink:label="lab_us-gaap_RelatedPartyDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RelatedPartyDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RelatedPartyDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RelatedPartyDomain" xlink:to="lab_us-gaap_RelatedPartyDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LoanreceivablefromaffiliatedentityrelatedtowarehousefacilityMember_edbf335d-6d3f-406c-a983-c41773e3f8a1_terseLabel_en-US" xlink:label="lab_efc_LoanreceivablefromaffiliatedentityrelatedtowarehousefacilityMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loan receivable from affiliated entity related to warehouse facility</link:label>
    <link:label id="lab_efc_LoanreceivablefromaffiliatedentityrelatedtowarehousefacilityMember_label_en-US" xlink:label="lab_efc_LoanreceivablefromaffiliatedentityrelatedtowarehousefacilityMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loan receivable from affiliated entity related to warehouse facility [Member]</link:label>
    <link:label id="lab_efc_LoanreceivablefromaffiliatedentityrelatedtowarehousefacilityMember_documentation_en-US" xlink:label="lab_efc_LoanreceivablefromaffiliatedentityrelatedtowarehousefacilityMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loan receivable from affiliated entity related to warehouse facility [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LoanreceivablefromaffiliatedentityrelatedtowarehousefacilityMember" xlink:href="efc-20201231.xsd#efc_LoanreceivablefromaffiliatedentityrelatedtowarehousefacilityMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LoanreceivablefromaffiliatedentityrelatedtowarehousefacilityMember" xlink:to="lab_efc_LoanreceivablefromaffiliatedentityrelatedtowarehousefacilityMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Increasedecreaseinaccruedmanagementfee_6dca9445-ed49-4e5d-b563-8ae3043bd989_terseLabel_en-US" xlink:label="lab_efc_Increasedecreaseinaccruedmanagementfee" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Base management fee payable to affiliate</link:label>
    <link:label id="lab_efc_Increasedecreaseinaccruedmanagementfee_label_en-US" xlink:label="lab_efc_Increasedecreaseinaccruedmanagementfee" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase decrease in accrued management fee</link:label>
    <link:label id="lab_efc_Increasedecreaseinaccruedmanagementfee_documentation_en-US" xlink:label="lab_efc_Increasedecreaseinaccruedmanagementfee" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (decrease) in the amount accrued and payable to the Manager for management fees.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Increasedecreaseinaccruedmanagementfee" xlink:href="efc-20201231.xsd#efc_Increasedecreaseinaccruedmanagementfee"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Increasedecreaseinaccruedmanagementfee" xlink:to="lab_efc_Increasedecreaseinaccruedmanagementfee" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationThreePointFivePercentMember_b5561450-84ea-44d9-aada-1ab6c22c75bd_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Three Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationThreePointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Three Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationThreePointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Three Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationThreePointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationThreePointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationThreePointFivePercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationThreePointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Notionalvalueofpurchasedcreditderivativesoffsettingwrittencreditderivatives_403842f5-13f9-4647-a469-cb76da6df3ed_negatedTerseLabel_en-US" xlink:label="lab_efc_Notionalvalueofpurchasedcreditderivativesoffsettingwrittencreditderivatives" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Notional value of purchased credit derivatives offsetting written credit derivatives</link:label>
    <link:label id="lab_efc_Notionalvalueofpurchasedcreditderivativesoffsettingwrittencreditderivatives_label_en-US" xlink:label="lab_efc_Notionalvalueofpurchasedcreditderivativesoffsettingwrittencreditderivatives" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Notional value of purchased credit derivatives offsetting written credit derivatives</link:label>
    <link:label id="lab_efc_Notionalvalueofpurchasedcreditderivativesoffsettingwrittencreditderivatives_documentation_en-US" xlink:label="lab_efc_Notionalvalueofpurchasedcreditderivativesoffsettingwrittencreditderivatives" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Notional value of purchased credit derivatives (short or purchased protection) offsetting written credit derivatives (long or sold protection).</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Notionalvalueofpurchasedcreditderivativesoffsettingwrittencreditderivatives" xlink:href="efc-20201231.xsd#efc_Notionalvalueofpurchasedcreditderivativesoffsettingwrittencreditderivatives"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Notionalvalueofpurchasedcreditderivativesoffsettingwrittencreditderivatives" xlink:to="lab_efc_Notionalvalueofpurchasedcreditderivativesoffsettingwrittencreditderivatives" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CostApproachValuationTechniqueMember_dbc56889-e7bf-4dc6-9b8a-aeab7ac3e591_terseLabel_en-US" xlink:label="lab_us-gaap_CostApproachValuationTechniqueMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Valuation, Cost Approach</link:label>
    <link:label id="lab_us-gaap_CostApproachValuationTechniqueMember_label_en-US" xlink:label="lab_us-gaap_CostApproachValuationTechniqueMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Valuation, Cost Approach [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CostApproachValuationTechniqueMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CostApproachValuationTechniqueMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CostApproachValuationTechniqueMember" xlink:to="lab_us-gaap_CostApproachValuationTechniqueMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativesFairValueLineItems_896e678b-d3df-4cf4-9b44-21c71d5f8c49_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativesFairValueLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivatives, Fair Value [Line Items]</link:label>
    <link:label id="lab_us-gaap_DerivativesFairValueLineItems_label_en-US" xlink:label="lab_us-gaap_DerivativesFairValueLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivatives, Fair Value [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativesFairValueLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativesFairValueLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativesFairValueLineItems" xlink:to="lab_us-gaap_DerivativesFairValueLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AvailableForSaleSecuritiesDebtSecurities_1db5e8b0-1b10-431e-9143-65058a2da2c2_terseLabel_en-US" xlink:label="lab_us-gaap_AvailableForSaleSecuritiesDebtSecurities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities, at fair value(1)(2)</link:label>
    <link:label id="lab_us-gaap_AvailableForSaleSecuritiesDebtSecurities_label_en-US" xlink:label="lab_us-gaap_AvailableForSaleSecuritiesDebtSecurities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AvailableForSaleSecuritiesDebtSecurities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AvailableForSaleSecuritiesDebtSecurities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AvailableForSaleSecuritiesDebtSecurities" xlink:to="lab_us-gaap_AvailableForSaleSecuritiesDebtSecurities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LendSureMortgageCorpMember_a3d6db42-d49f-45a7-9e25-e77d837586fb_terseLabel_en-US" xlink:label="lab_efc_LendSureMortgageCorpMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">LendSure Mortgage Corp [Member]</link:label>
    <link:label id="lab_efc_LendSureMortgageCorpMember_label_en-US" xlink:label="lab_efc_LendSureMortgageCorpMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">LendSure Mortgage Corp [Member]</link:label>
    <link:label id="lab_efc_LendSureMortgageCorpMember_documentation_en-US" xlink:label="lab_efc_LendSureMortgageCorpMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">LendSure Mortgage Corp [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LendSureMortgageCorpMember" xlink:href="efc-20201231.xsd#efc_LendSureMortgageCorpMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LendSureMortgageCorpMember" xlink:to="lab_efc_LendSureMortgageCorpMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EffectiveIncomeTaxRateReconciliationMinorityInterestIncomeExpense_8c10d3c2-c36a-4d59-9121-f38d09e2644c_terseLabel_en-US" xlink:label="lab_us-gaap_EffectiveIncomeTaxRateReconciliationMinorityInterestIncomeExpense" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Effective Income Tax Rate Reconciliation, Noncontrolling Interest Income (Loss), Percent</link:label>
    <link:label id="lab_us-gaap_EffectiveIncomeTaxRateReconciliationMinorityInterestIncomeExpense_label_en-US" xlink:label="lab_us-gaap_EffectiveIncomeTaxRateReconciliationMinorityInterestIncomeExpense" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Effective Income Tax Rate Reconciliation, Noncontrolling Interest Income (Loss), Percent</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EffectiveIncomeTaxRateReconciliationMinorityInterestIncomeExpense" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EffectiveIncomeTaxRateReconciliationMinorityInterestIncomeExpense"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EffectiveIncomeTaxRateReconciliationMinorityInterestIncomeExpense" xlink:to="lab_us-gaap_EffectiveIncomeTaxRateReconciliationMinorityInterestIncomeExpense" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Fairvalueofloansheldinrelatedpartytrust_fdf36d60-1548-4a85-acc8-d31225c31aa0_terseLabel_en-US" xlink:label="lab_efc_Fairvalueofloansheldinrelatedpartytrust" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair value of loans held in related party trust</link:label>
    <link:label id="lab_efc_Fairvalueofloansheldinrelatedpartytrust_label_en-US" xlink:label="lab_efc_Fairvalueofloansheldinrelatedpartytrust" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair value of loans held in related party trust</link:label>
    <link:label id="lab_efc_Fairvalueofloansheldinrelatedpartytrust_documentation_en-US" xlink:label="lab_efc_Fairvalueofloansheldinrelatedpartytrust" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair value of loans held in related party trust for which the Company has a beneficial interest in the residual cash flows of the underlying loans held by the trust.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Fairvalueofloansheldinrelatedpartytrust" xlink:href="efc-20201231.xsd#efc_Fairvalueofloansheldinrelatedpartytrust"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Fairvalueofloansheldinrelatedpartytrust" xlink:to="lab_efc_Fairvalueofloansheldinrelatedpartytrust" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RetainedEarningsAccumulatedDeficit_0240ff13-7dea-4822-a9d0-d9d3ee869d9f_terseLabel_en-US" xlink:label="lab_us-gaap_RetainedEarningsAccumulatedDeficit" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Retained earnings (accumulated deficit)</link:label>
    <link:label id="lab_us-gaap_RetainedEarningsAccumulatedDeficit_label_en-US" xlink:label="lab_us-gaap_RetainedEarningsAccumulatedDeficit" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Retained Earnings (Accumulated Deficit)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RetainedEarningsAccumulatedDeficit" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RetainedEarningsAccumulatedDeficit"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RetainedEarningsAccumulatedDeficit" xlink:to="lab_us-gaap_RetainedEarningsAccumulatedDeficit" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PaymentsForRepurchaseOfCommonStock_15cdef4e-5304-4e78-baf5-9bea7e5becbb_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_PaymentsForRepurchaseOfCommonStock" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase of common stock</link:label>
    <link:label id="lab_us-gaap_PaymentsForRepurchaseOfCommonStock_label_en-US" xlink:label="lab_us-gaap_PaymentsForRepurchaseOfCommonStock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments for Repurchase of Common Stock</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PaymentsForRepurchaseOfCommonStock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsForRepurchaseOfCommonStock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PaymentsForRepurchaseOfCommonStock" xlink:to="lab_us-gaap_PaymentsForRepurchaseOfCommonStock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ReceivableForSecuritiesSoldMember_d70b4a98-5658-4fff-ba05-18008c87ce01_terseLabel_en-US" xlink:label="lab_efc_ReceivableForSecuritiesSoldMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Receivable For Securities Sold [Member]</link:label>
    <link:label id="lab_efc_ReceivableForSecuritiesSoldMember_label_en-US" xlink:label="lab_efc_ReceivableForSecuritiesSoldMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Receivable For Securities Sold [Member]</link:label>
    <link:label id="lab_efc_ReceivableForSecuritiesSoldMember_documentation_en-US" xlink:label="lab_efc_ReceivableForSecuritiesSoldMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Receivable For Securities Sold [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ReceivableForSecuritiesSoldMember" xlink:href="efc-20201231.xsd#efc_ReceivableForSecuritiesSoldMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ReceivableForSecuritiesSoldMember" xlink:to="lab_efc_ReceivableForSecuritiesSoldMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LongbridgeFinancialLLCMember_dea2cf4f-04f0-4ed9-a713-a7a2eab00056_terseLabel_en-US" xlink:label="lab_efc_LongbridgeFinancialLLCMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Longbridge Financial LLC [Member]</link:label>
    <link:label id="lab_efc_LongbridgeFinancialLLCMember_label_en-US" xlink:label="lab_efc_LongbridgeFinancialLLCMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Longbridge Financial LLC [Member]</link:label>
    <link:label id="lab_efc_LongbridgeFinancialLLCMember_documentation_en-US" xlink:label="lab_efc_LongbridgeFinancialLLCMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Longbridge Financial LLC [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LongbridgeFinancialLLCMember" xlink:href="efc-20201231.xsd#efc_LongbridgeFinancialLLCMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LongbridgeFinancialLLCMember" xlink:to="lab_efc_LongbridgeFinancialLLCMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetRealizedGainLoss_067aee75-7a7d-42e8-b7c5-1020f734d291_terseLabel_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetRealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets, Realized Gain (Loss)</link:label>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetRealizedGainLoss_label_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetRealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement With Unobservable Inputs Reconciliation, Recurring Basis, Asset, Realized Gain (Loss)</link:label>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetRealizedGainLoss_documentation_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetRealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized gain/loss on investment/derivatives classed as Level 3 within the fair value heirarchy.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetRealizedGainLoss" xlink:href="efc-20201231.xsd#efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetRealizedGainLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetRealizedGainLoss" xlink:to="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetRealizedGainLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityTaxIdentificationNumber_e8671fcc-9cef-4789-9c7f-4382f124a4f1_terseLabel_en-US" xlink:label="lab_dei_EntityTaxIdentificationNumber" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Tax Identification Number</link:label>
    <link:label id="lab_dei_EntityTaxIdentificationNumber_label_en-US" xlink:label="lab_dei_EntityTaxIdentificationNumber" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Tax Identification Number</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityTaxIdentificationNumber" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityTaxIdentificationNumber"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityTaxIdentificationNumber" xlink:to="lab_dei_EntityTaxIdentificationNumber" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FinancingFacilityTypeAxis_45fbb508-f149-4665-b601-a3282fdf6079_terseLabel_en-US" xlink:label="lab_efc_FinancingFacilityTypeAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Facility Type [Axis]</link:label>
    <link:label id="lab_efc_FinancingFacilityTypeAxis_label_en-US" xlink:label="lab_efc_FinancingFacilityTypeAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Facility Type [Axis]</link:label>
    <link:label id="lab_efc_FinancingFacilityTypeAxis_documentation_en-US" xlink:label="lab_efc_FinancingFacilityTypeAxis" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Facility Type [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FinancingFacilityTypeAxis" xlink:href="efc-20201231.xsd#efc_FinancingFacilityTypeAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FinancingFacilityTypeAxis" xlink:to="lab_efc_FinancingFacilityTypeAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EarningsPerShareDiluted_4a831c8f-28ad-47ad-b5ce-b4d0ed4a0585_totalLabel_en-US" xlink:label="lab_us-gaap_EarningsPerShareDiluted" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Diluted earnings per common share (in usd per share)</link:label>
    <link:label id="lab_us-gaap_EarningsPerShareDiluted_label_en-US" xlink:label="lab_us-gaap_EarningsPerShareDiluted" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share, Diluted</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EarningsPerShareDiluted" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EarningsPerShareDiluted"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EarningsPerShareDiluted" xlink:to="lab_us-gaap_EarningsPerShareDiluted" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Subordinatednotesissuedinsecuritizationtransaction_47aace34-4499-4e12-b4c1-6b6e7cb99485_terseLabel_en-US" xlink:label="lab_efc_Subordinatednotesissuedinsecuritizationtransaction" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subordinated notes issued in securitization transaction</link:label>
    <link:label id="lab_efc_Subordinatednotesissuedinsecuritizationtransaction_label_en-US" xlink:label="lab_efc_Subordinatednotesissuedinsecuritizationtransaction" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subordinated notes issued in securitization transaction</link:label>
    <link:label id="lab_efc_Subordinatednotesissuedinsecuritizationtransaction_documentation_en-US" xlink:label="lab_efc_Subordinatednotesissuedinsecuritizationtransaction" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Original face amount of subordinated notes issued pursuant to the securitization.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Subordinatednotesissuedinsecuritizationtransaction" xlink:href="efc-20201231.xsd#efc_Subordinatednotesissuedinsecuritizationtransaction"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Subordinatednotesissuedinsecuritizationtransaction" xlink:to="lab_efc_Subordinatednotesissuedinsecuritizationtransaction" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ProceedsFromBrokersFinancingActivities_a34a9590-5251-4aa8-8161-117ee7c1382d_terseLabel_en-US" xlink:label="lab_efc_ProceedsFromBrokersFinancingActivities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Due from brokers, net</link:label>
    <link:label id="lab_efc_ProceedsFromBrokersFinancingActivities_label_en-US" xlink:label="lab_efc_ProceedsFromBrokersFinancingActivities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds From Brokers, Financing Activities</link:label>
    <link:label id="lab_efc_ProceedsFromBrokersFinancingActivities_documentation_en-US" xlink:label="lab_efc_ProceedsFromBrokersFinancingActivities" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds From Brokers, Financing Activities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ProceedsFromBrokersFinancingActivities" xlink:href="efc-20201231.xsd#efc_ProceedsFromBrokersFinancingActivities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ProceedsFromBrokersFinancingActivities" xlink:to="lab_efc_ProceedsFromBrokersFinancingActivities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_AnnualBaseManagementFeePercentage_181786ba-1e8a-4f83-b46c-50251724ed39_terseLabel_en-US" xlink:label="lab_efc_AnnualBaseManagementFeePercentage" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Annual base management fee percentage</link:label>
    <link:label id="lab_efc_AnnualBaseManagementFeePercentage_label_en-US" xlink:label="lab_efc_AnnualBaseManagementFeePercentage" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Annual Base Management Fee Percentage</link:label>
    <link:label id="lab_efc_AnnualBaseManagementFeePercentage_documentation_en-US" xlink:label="lab_efc_AnnualBaseManagementFeePercentage" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage used to calculate fees paid to the Manager as per the Managment Agreement.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_AnnualBaseManagementFeePercentage" xlink:href="efc-20201231.xsd#efc_AnnualBaseManagementFeePercentage"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_AnnualBaseManagementFeePercentage" xlink:to="lab_efc_AnnualBaseManagementFeePercentage" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SalariesAndWages_dd035568-6ab2-4464-864d-043ec98cb1f4_terseLabel_en-US" xlink:label="lab_us-gaap_SalariesAndWages" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Compensation expense</link:label>
    <link:label id="lab_us-gaap_SalariesAndWages_label_en-US" xlink:label="lab_us-gaap_SalariesAndWages" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Salary and Wage, NonOfficer, Excluding Cost of Good and Service Sold</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SalariesAndWages" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SalariesAndWages"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SalariesAndWages" xlink:to="lab_us-gaap_SalariesAndWages" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentsSoldNotYetPurchasedLineItems_fc08233e-ef2b-4a31-8216-cd56b0c0a8af_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentsSoldNotYetPurchasedLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments Sold, Not yet Purchased [Line Items]</link:label>
    <link:label id="lab_us-gaap_InvestmentsSoldNotYetPurchasedLineItems_label_en-US" xlink:label="lab_us-gaap_InvestmentsSoldNotYetPurchasedLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments Sold, Not yet Purchased [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentsSoldNotYetPurchasedLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentsSoldNotYetPurchasedLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentsSoldNotYetPurchasedLineItems" xlink:to="lab_us-gaap_InvestmentsSoldNotYetPurchasedLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_TradingSymbol_b2187a63-2de7-468e-8b13-9e256095ac99_terseLabel_en-US" xlink:label="lab_dei_TradingSymbol" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Trading Symbol</link:label>
    <link:label id="lab_dei_TradingSymbol_label_en-US" xlink:label="lab_dei_TradingSymbol" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Trading Symbol</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_TradingSymbol" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_TradingSymbol"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_TradingSymbol" xlink:to="lab_dei_TradingSymbol" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MinimumMember_2a2d9ded-ab02-436a-8b31-1ae7d55e4b3d_terseLabel_en-US" xlink:label="lab_srt_MinimumMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Minimum</link:label>
    <link:label id="lab_srt_MinimumMember_label_en-US" xlink:label="lab_srt_MinimumMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Minimum [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MinimumMember" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MinimumMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MinimumMember" xlink:to="lab_srt_MinimumMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RepurchaseOfSecuritiesSoldShortInvestingActivities_3c294a51-bab7-4be1-a92a-d06044b381f4_negatedTerseLabel_en-US" xlink:label="lab_efc_RepurchaseOfSecuritiesSoldShortInvestingActivities" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase of securities sold short</link:label>
    <link:label id="lab_efc_RepurchaseOfSecuritiesSoldShortInvestingActivities_label_en-US" xlink:label="lab_efc_RepurchaseOfSecuritiesSoldShortInvestingActivities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase Of Securities Sold Short, Investing Activities</link:label>
    <link:label id="lab_efc_RepurchaseOfSecuritiesSoldShortInvestingActivities_documentation_en-US" xlink:label="lab_efc_RepurchaseOfSecuritiesSoldShortInvestingActivities" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase Of Securities Sold Short, Investing Activities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RepurchaseOfSecuritiesSoldShortInvestingActivities" xlink:href="efc-20201231.xsd#efc_RepurchaseOfSecuritiesSoldShortInvestingActivities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RepurchaseOfSecuritiesSoldShortInvestingActivities" xlink:to="lab_efc_RepurchaseOfSecuritiesSoldShortInvestingActivities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ThresholdforexercisingOptionalRedemption_57cefae2-6e93-4e6c-94b6-c45384bdf1dd_terseLabel_en-US" xlink:label="lab_efc_ThresholdforexercisingOptionalRedemption" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Threshold for exercising Optional Redemption</link:label>
    <link:label id="lab_efc_ThresholdforexercisingOptionalRedemption_label_en-US" xlink:label="lab_efc_ThresholdforexercisingOptionalRedemption" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Threshold for exercising Optional Redemption</link:label>
    <link:label id="lab_efc_ThresholdforexercisingOptionalRedemption_documentation_en-US" xlink:label="lab_efc_ThresholdforexercisingOptionalRedemption" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Threshold under which the aggregate stated principal balance of non-QM loans in securitization trust must be in order to exercise the Optional Redemption prior to the two year anniversary of the Closing Date of the securitization.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ThresholdforexercisingOptionalRedemption" xlink:href="efc-20201231.xsd#efc_ThresholdforexercisingOptionalRedemption"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ThresholdforexercisingOptionalRedemption" xlink:to="lab_efc_ThresholdforexercisingOptionalRedemption" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MinorityInterest_0cce4aad-661e-4f70-9dde-f1901f1c7b16_terseLabel_en-US" xlink:label="lab_us-gaap_MinorityInterest" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling interests</link:label>
    <link:label id="lab_us-gaap_MinorityInterest_label_en-US" xlink:label="lab_us-gaap_MinorityInterest" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stockholders' Equity Attributable to Noncontrolling Interest</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MinorityInterest" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MinorityInterest"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MinorityInterest" xlink:to="lab_us-gaap_MinorityInterest" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProceedsFromEquityMethodInvestmentDividendsOrDistributionsReturnOfCapital_9ba34cda-0d9b-4774-ab9d-18e2329c492e_terseLabel_en-US" xlink:label="lab_us-gaap_ProceedsFromEquityMethodInvestmentDividendsOrDistributionsReturnOfCapital" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Distributions from investments in unconsolidated entities</link:label>
    <link:label id="lab_us-gaap_ProceedsFromEquityMethodInvestmentDividendsOrDistributionsReturnOfCapital_label_en-US" xlink:label="lab_us-gaap_ProceedsFromEquityMethodInvestmentDividendsOrDistributionsReturnOfCapital" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from Equity Method Investment, Distribution, Return of Capital</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProceedsFromEquityMethodInvestmentDividendsOrDistributionsReturnOfCapital" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromEquityMethodInvestmentDividendsOrDistributionsReturnOfCapital"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProceedsFromEquityMethodInvestmentDividendsOrDistributionsReturnOfCapital" xlink:to="lab_us-gaap_ProceedsFromEquityMethodInvestmentDividendsOrDistributionsReturnOfCapital" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RepurchaseAgreementsMember_01ba31f3-b5f9-4a93-9ca1-3559b98c671b_terseLabel_en-US" xlink:label="lab_us-gaap_RepurchaseAgreementsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase Agreements</link:label>
    <link:label id="lab_us-gaap_RepurchaseAgreementsMember_label_en-US" xlink:label="lab_us-gaap_RepurchaseAgreementsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase Agreements [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RepurchaseAgreementsMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RepurchaseAgreementsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RepurchaseAgreementsMember" xlink:to="lab_us-gaap_RepurchaseAgreementsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationOthernet_41e0f56a-8603-4acf-92f4-1cca9eaef612_terseLabel_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationOthernet" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Other, net</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationOthernet_label_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationOthernet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Other, net</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationOthernet_documentation_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationOthernet" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount of other income not separately disclosed reported by equity method investee.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationOthernet" xlink:href="efc-20201231.xsd#efc_EquityMethodInvestmentSummarizedFinancialInformationOthernet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationOthernet" xlink:to="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationOthernet" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ConsumerMember_c84a4321-d519-4c5b-b206-a055dee18ff2_terseLabel_en-US" xlink:label="lab_efc_ConsumerMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consumer [Member]</link:label>
    <link:label id="lab_efc_ConsumerMember_label_en-US" xlink:label="lab_efc_ConsumerMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consumer [Member]</link:label>
    <link:label id="lab_efc_ConsumerMember_documentation_en-US" xlink:label="lab_efc_ConsumerMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consumer [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ConsumerMember" xlink:href="efc-20201231.xsd#efc_ConsumerMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ConsumerMember" xlink:to="lab_efc_ConsumerMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTextBlock_62b49735-aea2-4903-bcea-e2a078110893_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Restrictions on Cash and Cash Equivalents</link:label>
    <link:label id="lab_us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Restrictions on Cash and Cash Equivalents [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTextBlock" xlink:to="lab_us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeVariableInterestRate_714567f0-b969-400b-a4b0-82797a473a4e_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeVariableInterestRate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Receive Rate</link:label>
    <link:label id="lab_us-gaap_DerivativeVariableInterestRate_label_en-US" xlink:label="lab_us-gaap_DerivativeVariableInterestRate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative, Variable Interest Rate</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeVariableInterestRate" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeVariableInterestRate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeVariableInterestRate" xlink:to="lab_us-gaap_DerivativeVariableInterestRate" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FixedRateResidentialMortgages30YearMember_159fb655-1f10-4d76-8cf4-8f4035a12455_terseLabel_en-US" xlink:label="lab_efc_FixedRateResidentialMortgages30YearMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">30-year fixed-rate mortgages</link:label>
    <link:label id="lab_efc_FixedRateResidentialMortgages30YearMember_label_en-US" xlink:label="lab_efc_FixedRateResidentialMortgages30YearMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed Rate Residential Mortgages, 30 Year [Member]</link:label>
    <link:label id="lab_efc_FixedRateResidentialMortgages30YearMember_documentation_en-US" xlink:label="lab_efc_FixedRateResidentialMortgages30YearMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed Rate Residential Mortgages, 30 Year [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FixedRateResidentialMortgages30YearMember" xlink:href="efc-20201231.xsd#efc_FixedRateResidentialMortgages30YearMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FixedRateResidentialMortgages30YearMember" xlink:to="lab_efc_FixedRateResidentialMortgages30YearMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetPurchases_70cd491a-5c6a-45e1-93b2-dcade5a6e597_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetPurchases" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets, Purchases</link:label>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetPurchases_label_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetPurchases" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement with Unobservable Inputs Reconciliation, Recurring Basis, Asset, Purchases</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetPurchases" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetPurchases"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetPurchases" xlink:to="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetPurchases" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_LegalEntityAxis_83a806b0-8868-4474-94c8-53bfc39bd9c0_terseLabel_en-US" xlink:label="lab_dei_LegalEntityAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Legal Entity [Axis]</link:label>
    <link:label id="lab_dei_LegalEntityAxis_label_en-US" xlink:label="lab_dei_LegalEntityAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Legal Entity [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_LegalEntityAxis" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_LegalEntityAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_LegalEntityAxis" xlink:to="lab_dei_LegalEntityAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ToBeAnnouncedSecuritiesTable_da091a16-999a-4168-8db5-efe7903e4905_terseLabel_en-US" xlink:label="lab_efc_ToBeAnnouncedSecuritiesTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">To Be Announced RMBS [Table]</link:label>
    <link:label id="lab_efc_ToBeAnnouncedSecuritiesTable_label_en-US" xlink:label="lab_efc_ToBeAnnouncedSecuritiesTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">To Be Announced Securities [Table]</link:label>
    <link:label id="lab_efc_ToBeAnnouncedSecuritiesTable_documentation_en-US" xlink:label="lab_efc_ToBeAnnouncedSecuritiesTable" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">To Be Announced Securities [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ToBeAnnouncedSecuritiesTable" xlink:href="efc-20201231.xsd#efc_ToBeAnnouncedSecuritiesTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ToBeAnnouncedSecuritiesTable" xlink:to="lab_efc_ToBeAnnouncedSecuritiesTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MortgageLoansOnRealEstateForeclosures_2e3e1699-a191-4192-a8f9-5672a6acf813_terseLabel_en-US" xlink:label="lab_us-gaap_MortgageLoansOnRealEstateForeclosures" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Foreclosure</link:label>
    <link:label id="lab_us-gaap_MortgageLoansOnRealEstateForeclosures_label_en-US" xlink:label="lab_us-gaap_MortgageLoansOnRealEstateForeclosures" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Foreclosure</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MortgageLoansOnRealEstateForeclosures" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MortgageLoansOnRealEstateForeclosures"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MortgageLoansOnRealEstateForeclosures" xlink:to="lab_us-gaap_MortgageLoansOnRealEstateForeclosures" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DeferredIncomeTaxExpenseBenefit_e70bd8a0-d8e1-4487-802b-28ff9cb23d68_terseLabel_en-US" xlink:label="lab_us-gaap_DeferredIncomeTaxExpenseBenefit" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Income Tax Expense (Benefit)</link:label>
    <link:label id="lab_us-gaap_DeferredIncomeTaxExpenseBenefit_label_en-US" xlink:label="lab_us-gaap_DeferredIncomeTaxExpenseBenefit" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Income Tax Expense (Benefit)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DeferredIncomeTaxExpenseBenefit" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DeferredIncomeTaxExpenseBenefit"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DeferredIncomeTaxExpenseBenefit" xlink:to="lab_us-gaap_DeferredIncomeTaxExpenseBenefit" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsLineItems_2e40c33f-5af4-40f9-842f-8c7531c0932d_terseLabel_en-US" xlink:label="lab_us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transfer of Certain Financial Assets Accounted for as Secured Borrowings [Line Items]</link:label>
    <link:label id="lab_us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsLineItems_label_en-US" xlink:label="lab_us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transfer of Certain Financial Assets Accounted for as Secured Borrowings [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsLineItems" xlink:to="lab_us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseGross_e5b9dcd3-a8d5-45c4-97cd-5c25dcd5c7e4_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseGross" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Sold under Agreements to Repurchase, Gross</link:label>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseGross_label_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseGross" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Sold under Agreements to Repurchase, Gross</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseGross" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseGross"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseGross" xlink:to="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseGross" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Companysinitialpercentageownershipofjointlyownedentity_6648c981-e685-4210-8719-3029efe6e145_terseLabel_en-US" xlink:label="lab_efc_Companysinitialpercentageownershipofjointlyownedentity" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Company's initial percentage ownership of jointly owned entity</link:label>
    <link:label id="lab_efc_Companysinitialpercentageownershipofjointlyownedentity_label_en-US" xlink:label="lab_efc_Companysinitialpercentageownershipofjointlyownedentity" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Company's initial percentage ownership of jointly owned entity</link:label>
    <link:label id="lab_efc_Companysinitialpercentageownershipofjointlyownedentity_documentation_en-US" xlink:label="lab_efc_Companysinitialpercentageownershipofjointlyownedentity" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Company's initial percentage ownership of jointly owned entity with affiliated co-participant.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Companysinitialpercentageownershipofjointlyownedentity" xlink:href="efc-20201231.xsd#efc_Companysinitialpercentageownershipofjointlyownedentity"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Companysinitialpercentageownershipofjointlyownedentity" xlink:to="lab_efc_Companysinitialpercentageownershipofjointlyownedentity" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AdjustmentsToReconcileNetIncomeLossToCashProvidedByUsedInOperatingActivitiesAbstract_47ea6f34-3a7e-4c2e-bd39-736befecb1bc_terseLabel_en-US" xlink:label="lab_us-gaap_AdjustmentsToReconcileNetIncomeLossToCashProvidedByUsedInOperatingActivitiesAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reconciliation of the net increase (decrease) in equity resulting from operations to net cash provided by (used in) operating activities:</link:label>
    <link:label id="lab_us-gaap_AdjustmentsToReconcileNetIncomeLossToCashProvidedByUsedInOperatingActivitiesAbstract_label_en-US" xlink:label="lab_us-gaap_AdjustmentsToReconcileNetIncomeLossToCashProvidedByUsedInOperatingActivitiesAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Adjustments to Reconcile Net Income (Loss) to Cash Provided by (Used in) Operating Activities [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AdjustmentsToReconcileNetIncomeLossToCashProvidedByUsedInOperatingActivitiesAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AdjustmentsToReconcileNetIncomeLossToCashProvidedByUsedInOperatingActivitiesAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AdjustmentsToReconcileNetIncomeLossToCashProvidedByUsedInOperatingActivitiesAbstract" xlink:to="lab_us-gaap_AdjustmentsToReconcileNetIncomeLossToCashProvidedByUsedInOperatingActivitiesAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsTable_74302470-198c-4aac-8da4-5368df885753_terseLabel_en-US" xlink:label="lab_us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transfer of Certain Financial Assets Accounted for as Secured Borrowings [Table]</link:label>
    <link:label id="lab_us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsTable_label_en-US" xlink:label="lab_us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transfer of Certain Financial Assets Accounted for as Secured Borrowings [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsTable" xlink:to="lab_us-gaap_TransferOfCertainFinancialAssetsAccountedForAsSecuredBorrowingsTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NetCashProvidedByUsedInFinancingActivities_9a5c7337-d641-4c13-8ebc-666c76af33e0_totalLabel_en-US" xlink:label="lab_us-gaap_NetCashProvidedByUsedInFinancingActivities" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net cash provided by (used in) financing activities</link:label>
    <link:label id="lab_us-gaap_NetCashProvidedByUsedInFinancingActivities_label_en-US" xlink:label="lab_us-gaap_NetCashProvidedByUsedInFinancingActivities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Cash Provided by (Used in) Financing Activities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NetCashProvidedByUsedInFinancingActivities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NetCashProvidedByUsedInFinancingActivities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NetCashProvidedByUsedInFinancingActivities" xlink:to="lab_us-gaap_NetCashProvidedByUsedInFinancingActivities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationFourPercentMember_d5a28c44-6312-40f6-b7f5-0fe12ee92ce3_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Four Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationFourPercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Four Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationFourPercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Four Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationFourPercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationFourPercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationFourPercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationFourPercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncomeTaxExpenseBenefit_9f30d164-7709-4047-aaab-635b08d037d8_terseLabel_en-US" xlink:label="lab_us-gaap_IncomeTaxExpenseBenefit" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Income Tax Expense (Benefit)</link:label>
    <link:label id="lab_us-gaap_IncomeTaxExpenseBenefit_label_en-US" xlink:label="lab_us-gaap_IncomeTaxExpenseBenefit" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Income Tax Expense (Benefit)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncomeTaxExpenseBenefit" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncomeTaxExpenseBenefit"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncomeTaxExpenseBenefit" xlink:to="lab_us-gaap_IncomeTaxExpenseBenefit" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ParticipationinMultiSellerConsumerLoanSecuritizationMember_3e51e32e-3a58-4cb4-91bb-fd86331ccbd8_terseLabel_en-US" xlink:label="lab_efc_ParticipationinMultiSellerConsumerLoanSecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Participation in Multi-Seller Consumer Loan Securitization [Member]</link:label>
    <link:label id="lab_efc_ParticipationinMultiSellerConsumerLoanSecuritizationMember_label_en-US" xlink:label="lab_efc_ParticipationinMultiSellerConsumerLoanSecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Participation in Multi-Seller Consumer Loan Securitization [Member]</link:label>
    <link:label id="lab_efc_ParticipationinMultiSellerConsumerLoanSecuritizationMember_documentation_en-US" xlink:label="lab_efc_ParticipationinMultiSellerConsumerLoanSecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Participation in Multi-Seller Consumer Loan Securitization [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ParticipationinMultiSellerConsumerLoanSecuritizationMember" xlink:href="efc-20201231.xsd#efc_ParticipationinMultiSellerConsumerLoanSecuritizationMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ParticipationinMultiSellerConsumerLoanSecuritizationMember" xlink:to="lab_efc_ParticipationinMultiSellerConsumerLoanSecuritizationMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_VestDecember172021Member_8f07d117-cff1-46da-b3b7-1038f746dd73_terseLabel_en-US" xlink:label="lab_efc_VestDecember172021Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 17, 2021</link:label>
    <link:label id="lab_efc_VestDecember172021Member_label_en-US" xlink:label="lab_efc_VestDecember172021Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 17, 2021 [Member]</link:label>
    <link:label id="lab_efc_VestDecember172021Member_documentation_en-US" xlink:label="lab_efc_VestDecember172021Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 17, 2021</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_VestDecember172021Member" xlink:href="efc-20201231.xsd#efc_VestDecember172021Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_VestDecember172021Member" xlink:to="lab_efc_VestDecember172021Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationEquity_188b4e8d-ec2d-4e5e-b804-710a0b99846e_terseLabel_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationEquity" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment Summarized Financial Information, Equity</link:label>
    <link:label id="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationEquity_label_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationEquity" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment Summarized Financial Information, Equity</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationEquity" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquityMethodInvestmentSummarizedFinancialInformationEquity"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationEquity" xlink:to="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationEquity" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_VestingAxis_ee332796-facb-412c-a79d-a76cda355d2d_terseLabel_en-US" xlink:label="lab_us-gaap_VestingAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vesting [Axis]</link:label>
    <link:label id="lab_us-gaap_VestingAxis_label_en-US" xlink:label="lab_us-gaap_VestingAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vesting [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_VestingAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_VestingAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_VestingAxis" xlink:to="lab_us-gaap_VestingAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProceedsFromSaleOfLoansReceivable_93b82aa6-5d9b-4efe-ac12-0d33bde5f470_terseLabel_en-US" xlink:label="lab_us-gaap_ProceedsFromSaleOfLoansReceivable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from disposition of loans</link:label>
    <link:label id="lab_us-gaap_ProceedsFromSaleOfLoansReceivable_label_en-US" xlink:label="lab_us-gaap_ProceedsFromSaleOfLoansReceivable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from Sale of Loans Receivable</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProceedsFromSaleOfLoansReceivable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromSaleOfLoansReceivable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProceedsFromSaleOfLoansReceivable" xlink:to="lab_us-gaap_ProceedsFromSaleOfLoansReceivable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuredborrowingfacilitycollateralizedbyABSbackedbyconsumerloansMember_787a87df-d010-40c6-9b75-207d2b88163c_terseLabel_en-US" xlink:label="lab_efc_SecuredborrowingfacilitycollateralizedbyABSbackedbyconsumerloansMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Secured borrowing facility collateralized by ABS backed by consumer loans [Member]</link:label>
    <link:label id="lab_efc_SecuredborrowingfacilitycollateralizedbyABSbackedbyconsumerloansMember_label_en-US" xlink:label="lab_efc_SecuredborrowingfacilitycollateralizedbyABSbackedbyconsumerloansMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Secured borrowing facility collateralized by ABS backed by consumer loans [Member]</link:label>
    <link:label id="lab_efc_SecuredborrowingfacilitycollateralizedbyABSbackedbyconsumerloansMember_documentation_en-US" xlink:label="lab_efc_SecuredborrowingfacilitycollateralizedbyABSbackedbyconsumerloansMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Secured borrowing facility collateralized by ABS backed by consumer loans [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuredborrowingfacilitycollateralizedbyABSbackedbyconsumerloansMember" xlink:href="efc-20201231.xsd#efc_SecuredborrowingfacilitycollateralizedbyABSbackedbyconsumerloansMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuredborrowingfacilitycollateralizedbyABSbackedbyconsumerloansMember" xlink:to="lab_efc_SecuredborrowingfacilitycollateralizedbyABSbackedbyconsumerloansMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeFairValueOfDerivativeLiability_564d1429-59c4-48b2-817f-d59873f2aaef_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeFairValueOfDerivativeLiability" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Liability, Fair Value, Gross Liability</link:label>
    <link:label id="lab_us-gaap_DerivativeFairValueOfDerivativeLiability_label_en-US" xlink:label="lab_us-gaap_DerivativeFairValueOfDerivativeLiability" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Liability, Fair Value, Gross Liability</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeFairValueOfDerivativeLiability" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeFairValueOfDerivativeLiability"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeFairValueOfDerivativeLiability" xlink:to="lab_us-gaap_DerivativeFairValueOfDerivativeLiability" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NetAssetValuePerShare_92d67e9a-5d26-423d-bee3-ad5118ebbf75_terseLabel_en-US" xlink:label="lab_us-gaap_NetAssetValuePerShare" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common shares</link:label>
    <link:label id="lab_us-gaap_NetAssetValuePerShare_b02614cc-07d9-45ca-ac9e-b798dff68b45_periodStartLabel_en-US" xlink:label="lab_us-gaap_NetAssetValuePerShare" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Asset Value Per Share, Beginning Balance</link:label>
    <link:label id="lab_us-gaap_NetAssetValuePerShare_04356e3b-8288-48fe-b64a-e236c5c455ec_periodEndLabel_en-US" xlink:label="lab_us-gaap_NetAssetValuePerShare" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Asset Value Per Share, Ending Balance</link:label>
    <link:label id="lab_us-gaap_NetAssetValuePerShare_label_en-US" xlink:label="lab_us-gaap_NetAssetValuePerShare" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Asset Value Per Share</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NetAssetValuePerShare" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NetAssetValuePerShare"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NetAssetValuePerShare" xlink:to="lab_us-gaap_NetAssetValuePerShare" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Distributionstononcontrollinginterests_4cdf6bac-dbd5-4a24-929c-fc012c25ca77_terseLabel_en-US" xlink:label="lab_efc_Distributionstononcontrollinginterests" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Distributions to non-controlling interests</link:label>
    <link:label id="lab_efc_Distributionstononcontrollinginterests_label_en-US" xlink:label="lab_efc_Distributionstononcontrollinginterests" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Distributions to non-controlling interests</link:label>
    <link:label id="lab_efc_Distributionstononcontrollinginterests_documentation_en-US" xlink:label="lab_efc_Distributionstononcontrollinginterests" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Distributions to non-controlling interests</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Distributionstononcontrollinginterests" xlink:href="efc-20201231.xsd#efc_Distributionstononcontrollinginterests"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Distributionstononcontrollinginterests" xlink:to="lab_efc_Distributionstononcontrollinginterests" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_CityAreaCode_13f85d0c-93cd-487e-b2ad-88f404e66cc8_terseLabel_en-US" xlink:label="lab_dei_CityAreaCode" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">City Area Code</link:label>
    <link:label id="lab_dei_CityAreaCode_label_en-US" xlink:label="lab_dei_CityAreaCode" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">City Area Code</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_CityAreaCode" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_CityAreaCode"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_CityAreaCode" xlink:to="lab_dei_CityAreaCode" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_TBAsecuritiesMember_af35b61b-bd78-4e0a-a945-7de47dbf8cce_terseLabel_en-US" xlink:label="lab_efc_TBAsecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">TBA securities</link:label>
    <link:label id="lab_efc_TBAsecuritiesMember_label_en-US" xlink:label="lab_efc_TBAsecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">TBA securities [Member]</link:label>
    <link:label id="lab_efc_TBAsecuritiesMember_documentation_en-US" xlink:label="lab_efc_TBAsecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Forward contract for the purchase or sale of Agency RMBS in the To-be-announced RMBS market.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_TBAsecuritiesMember" xlink:href="efc-20201231.xsd#efc_TBAsecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_TBAsecuritiesMember" xlink:to="lab_efc_TBAsecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseRepurchaseLiability_6c6af9c4-6eb9-4f0d-bf2d-cf71d7113e52_terseLabel_en-US" xlink:label="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseRepurchaseLiability" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase agreements</link:label>
    <link:label id="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseRepurchaseLiability_label_en-US" xlink:label="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseRepurchaseLiability" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets Sold under Agreements to Repurchase, Repurchase Liability</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AssetsSoldUnderAgreementsToRepurchaseRepurchaseLiability" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AssetsSoldUnderAgreementsToRepurchaseRepurchaseLiability"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AssetsSoldUnderAgreementsToRepurchaseRepurchaseLiability" xlink:to="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseRepurchaseLiability" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPercentMember_36b0f7ff-865e-4c62-9b36-0b90be2c9589_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Four Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Four Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Four Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_TypeOfRecipientDomain_c7536ed9-9143-48a1-96e0-937512b88943_terseLabel_en-US" xlink:label="lab_efc_TypeOfRecipientDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Type of Recipient [Domain]</link:label>
    <link:label id="lab_efc_TypeOfRecipientDomain_label_en-US" xlink:label="lab_efc_TypeOfRecipientDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Type Of Recipient [Domain]</link:label>
    <link:label id="lab_efc_TypeOfRecipientDomain_documentation_en-US" xlink:label="lab_efc_TypeOfRecipientDomain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disclosure by type of recipient related to share-based payment arrangements (LTIP awards).</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_TypeOfRecipientDomain" xlink:href="efc-20201231.xsd#efc_TypeOfRecipientDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_TypeOfRecipientDomain" xlink:to="lab_efc_TypeOfRecipientDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_Security12bTitle_595778ee-009b-495c-922b-996038eefae7_terseLabel_en-US" xlink:label="lab_dei_Security12bTitle" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Title of 12(b) Security</link:label>
    <link:label id="lab_dei_Security12bTitle_label_en-US" xlink:label="lab_dei_Security12bTitle" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Title of 12(b) Security</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_Security12bTitle" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_Security12bTitle"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_Security12bTitle" xlink:to="lab_dei_Security12bTitle" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncreaseDecreaseInAccruedInterestReceivableNet_30503fc7-911d-4ec8-a6b4-fecdc35941e7_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInAccruedInterestReceivableNet" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest and principal receivable</link:label>
    <link:label id="lab_us-gaap_IncreaseDecreaseInAccruedInterestReceivableNet_label_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInAccruedInterestReceivableNet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (Decrease) in Accrued Interest Receivable, Net</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncreaseDecreaseInAccruedInterestReceivableNet" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncreaseDecreaseInAccruedInterestReceivableNet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncreaseDecreaseInAccruedInterestReceivableNet" xlink:to="lab_us-gaap_IncreaseDecreaseInAccruedInterestReceivableNet" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationFivePercentMember_5a1f63a8-fee3-4880-9bc1-aba7af8e11ec_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationFivePercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationFivePercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationFivePercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationFivePercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DerivativesProceeds_5a032b00-34e0-4268-9e90-b5568df7dfac_terseLabel_en-US" xlink:label="lab_efc_DerivativesProceeds" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial derivatives&#8211;assets, at fair value-Net Proceeds</link:label>
    <link:label id="lab_efc_DerivativesProceeds_fbf7c6f9-143a-4c39-9f8c-860147bdd715_negatedTerseLabel_en-US" xlink:label="lab_efc_DerivativesProceeds" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivatives Proceeds</link:label>
    <link:label id="lab_efc_DerivativesProceeds_label_en-US" xlink:label="lab_efc_DerivativesProceeds" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivatives Proceeds</link:label>
    <link:label id="lab_efc_DerivativesProceeds_documentation_en-US" xlink:label="lab_efc_DerivativesProceeds" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Represents cost basis of derivative liabilities carried at fair value.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DerivativesProceeds" xlink:href="efc-20201231.xsd#efc_DerivativesProceeds"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DerivativesProceeds" xlink:to="lab_efc_DerivativesProceeds" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AccountingGuidanceMember_a38ebd15-4ddb-4a39-a595-05346e9d4865_terseLabel_en-US" xlink:label="lab_us-gaap_AccountingGuidanceMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accounting Guidance [Domain]</link:label>
    <link:label id="lab_us-gaap_AccountingGuidanceMember_label_en-US" xlink:label="lab_us-gaap_AccountingGuidanceMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accounting Guidance [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AccountingGuidanceMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AccountingGuidanceMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AccountingGuidanceMember" xlink:to="lab_us-gaap_AccountingGuidanceMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SharesIssuedAsPaymentofIncentiveFeeShares_4d016810-7056-48ad-87e6-9a847dcc308e_terseLabel_en-US" xlink:label="lab_efc_SharesIssuedAsPaymentofIncentiveFeeShares" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Shares Issued As Payment of Incentive Fee, Shares</link:label>
    <link:label id="lab_efc_SharesIssuedAsPaymentofIncentiveFeeShares_label_en-US" xlink:label="lab_efc_SharesIssuedAsPaymentofIncentiveFeeShares" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Shares Issued As Payment of Incentive Fee, Shares</link:label>
    <link:label id="lab_efc_SharesIssuedAsPaymentofIncentiveFeeShares_documentation_en-US" xlink:label="lab_efc_SharesIssuedAsPaymentofIncentiveFeeShares" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of common shares issued to affiliate as partial payment for performance fees.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SharesIssuedAsPaymentofIncentiveFeeShares" xlink:href="efc-20201231.xsd#efc_SharesIssuedAsPaymentofIncentiveFeeShares"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SharesIssuedAsPaymentofIncentiveFeeShares" xlink:to="lab_efc_SharesIssuedAsPaymentofIncentiveFeeShares" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AssetsFairValueDisclosure_d1480c57-a2c4-4fd3-bb0e-b7d146b9bb47_totalLabel_en-US" xlink:label="lab_us-gaap_AssetsFairValueDisclosure" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total assets</link:label>
    <link:label id="lab_us-gaap_AssetsFairValueDisclosure_3e620228-6ba4-4b8c-8637-0f6b53b6ea7d_verboseLabel_en-US" xlink:label="lab_us-gaap_AssetsFairValueDisclosure" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair value</link:label>
    <link:label id="lab_us-gaap_AssetsFairValueDisclosure_label_en-US" xlink:label="lab_us-gaap_AssetsFairValueDisclosure" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets, Fair Value Disclosure</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AssetsFairValueDisclosure" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AssetsFairValueDisclosure"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AssetsFairValueDisclosure" xlink:to="lab_us-gaap_AssetsFairValueDisclosure" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CommonStockCapitalSharesReservedForFutureIssuance_d007cfa0-7294-4bde-b0b5-2393e01ea2b2_terseLabel_en-US" xlink:label="lab_us-gaap_CommonStockCapitalSharesReservedForFutureIssuance" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Capital shares reserved for future issuance (in shares)</link:label>
    <link:label id="lab_us-gaap_CommonStockCapitalSharesReservedForFutureIssuance_label_en-US" xlink:label="lab_us-gaap_CommonStockCapitalSharesReservedForFutureIssuance" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common Stock, Capital Shares Reserved for Future Issuance</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CommonStockCapitalSharesReservedForFutureIssuance" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CommonStockCapitalSharesReservedForFutureIssuance"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CommonStockCapitalSharesReservedForFutureIssuance" xlink:to="lab_us-gaap_CommonStockCapitalSharesReservedForFutureIssuance" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_BarclaysCapitalIncCollateralizedbyParValue800ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member_5f39e2d2-88cf-4f3a-92fa-2df87364926c_terseLabel_en-US" xlink:label="lab_efc_BarclaysCapitalIncCollateralizedbyParValue800ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Barclays Capital Inc Collateralized by Par Value $800 Exchange-Traded Corporate Debt, Coupon 9.88%, Maturity Date 2024 02 [Member]</link:label>
    <link:label id="lab_efc_BarclaysCapitalIncCollateralizedbyParValue800ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member_label_en-US" xlink:label="lab_efc_BarclaysCapitalIncCollateralizedbyParValue800ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Barclays Capital Inc Collateralized by Par Value $800 Exchange-Traded Corporate Debt, Coupon 9.88%, Maturity Date 2024 02 [Member]</link:label>
    <link:label id="lab_efc_BarclaysCapitalIncCollateralizedbyParValue800ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member_documentation_en-US" xlink:label="lab_efc_BarclaysCapitalIncCollateralizedbyParValue800ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Barclays Capital Inc Collateralized by Par Value $800 Exchange-Traded Corporate Debt, Coupon 9.88%, Maturity Date 2024 02 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_BarclaysCapitalIncCollateralizedbyParValue800ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member" xlink:href="efc-20201231.xsd#efc_BarclaysCapitalIncCollateralizedbyParValue800ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_BarclaysCapitalIncCollateralizedbyParValue800ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member" xlink:to="lab_efc_BarclaysCapitalIncCollateralizedbyParValue800ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GreaterthanelevenyearMember_c6721e11-edbc-4e84-9631-4bc00a2c0077_terseLabel_en-US" xlink:label="lab_efc_GreaterthanelevenyearMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Greater than eleven year [Member]</link:label>
    <link:label id="lab_efc_GreaterthanelevenyearMember_label_en-US" xlink:label="lab_efc_GreaterthanelevenyearMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Greater than eleven year [Member]</link:label>
    <link:label id="lab_efc_GreaterthanelevenyearMember_documentation_en-US" xlink:label="lab_efc_GreaterthanelevenyearMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Greater than eleven year [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GreaterthanelevenyearMember" xlink:href="efc-20201231.xsd#efc_GreaterthanelevenyearMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GreaterthanelevenyearMember" xlink:to="lab_efc_GreaterthanelevenyearMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancingReceivablePercentPastDue1_0c3e2685-5a5d-45cd-8f1b-1166ec8b9085_terseLabel_en-US" xlink:label="lab_us-gaap_FinancingReceivablePercentPastDue1" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percent past due</link:label>
    <link:label id="lab_us-gaap_FinancingReceivablePercentPastDue1_label_en-US" xlink:label="lab_us-gaap_FinancingReceivablePercentPastDue1" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable, Percent Past Due</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancingReceivablePercentPastDue1" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancingReceivablePercentPastDue1"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancingReceivablePercentPastDue1" xlink:to="lab_us-gaap_FinancingReceivablePercentPastDue1" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FloatingRateAgencySecuritiesMember_b4c0f0ff-5bcf-4fad-b990-985e36c6b46e_terseLabel_en-US" xlink:label="lab_efc_FloatingRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Floating Rate Agency Securities [Member]</link:label>
    <link:label id="lab_efc_FloatingRateAgencySecuritiesMember_label_en-US" xlink:label="lab_efc_FloatingRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Floating Rate Agency Securities [Member]</link:label>
    <link:label id="lab_efc_FloatingRateAgencySecuritiesMember_documentation_en-US" xlink:label="lab_efc_FloatingRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">A security issued by a government agency or government sponsored enterprise which has a floating interest rate.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FloatingRateAgencySecuritiesMember" xlink:href="efc-20201231.xsd#efc_FloatingRateAgencySecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FloatingRateAgencySecuritiesMember" xlink:to="lab_efc_FloatingRateAgencySecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProceedsFromIssuanceOfSecuredDebt_779cf3a3-644d-415d-9148-dab1a778ebe4_terseLabel_en-US" xlink:label="lab_us-gaap_ProceedsFromIssuanceOfSecuredDebt" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from issuance of Other secured borrowings</link:label>
    <link:label id="lab_us-gaap_ProceedsFromIssuanceOfSecuredDebt_label_en-US" xlink:label="lab_us-gaap_ProceedsFromIssuanceOfSecuredDebt" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from Issuance of Secured Debt</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProceedsFromIssuanceOfSecuredDebt" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromIssuanceOfSecuredDebt"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProceedsFromIssuanceOfSecuredDebt" xlink:to="lab_us-gaap_ProceedsFromIssuanceOfSecuredDebt" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue1355U.S.TreasuryNoteCoupon2.75MaturityDate202304Member_7f5c9294-9503-450f-b69b-bb65ad29fa25_terseLabel_en-US" xlink:label="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue1355U.S.TreasuryNoteCoupon2.75MaturityDate202304Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of America Securities Collateralized by Par Value $1,355 U.S. Treasury Note, Coupon 2.75%, Maturity Date 2023 04 [Member]</link:label>
    <link:label id="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue1355U.S.TreasuryNoteCoupon2.75MaturityDate202304Member_label_en-US" xlink:label="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue1355U.S.TreasuryNoteCoupon2.75MaturityDate202304Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of America Securities Collateralized by Par Value $1,355 U.S. Treasury Note, Coupon 2.75%, Maturity Date 2023 04 [Member]</link:label>
    <link:label id="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue1355U.S.TreasuryNoteCoupon2.75MaturityDate202304Member_documentation_en-US" xlink:label="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue1355U.S.TreasuryNoteCoupon2.75MaturityDate202304Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of America Securities Collateralized by Par Value $1,355 U.S. Treasury Note, Coupon 2.75%, Maturity Date 2023 04 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_BankofAmericaSecuritiesCollateralizedbyParValue1355U.S.TreasuryNoteCoupon2.75MaturityDate202304Member" xlink:href="efc-20201231.xsd#efc_BankofAmericaSecuritiesCollateralizedbyParValue1355U.S.TreasuryNoteCoupon2.75MaturityDate202304Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_BankofAmericaSecuritiesCollateralizedbyParValue1355U.S.TreasuryNoteCoupon2.75MaturityDate202304Member" xlink:to="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue1355U.S.TreasuryNoteCoupon2.75MaturityDate202304Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ConcentrationRiskTable_706b2b88-1beb-44d5-b303-e9b8dc122dd1_terseLabel_en-US" xlink:label="lab_us-gaap_ConcentrationRiskTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk [Table]</link:label>
    <link:label id="lab_us-gaap_ConcentrationRiskTable_label_en-US" xlink:label="lab_us-gaap_ConcentrationRiskTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ConcentrationRiskTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ConcentrationRiskTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ConcentrationRiskTable" xlink:to="lab_us-gaap_ConcentrationRiskTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_IncreaseDecreaseInDueToBrokersMarginAccounts_4732bf3a-bfce-4edd-b5d9-bbe6b9e4e11f_terseLabel_en-US" xlink:label="lab_efc_IncreaseDecreaseInDueToBrokersMarginAccounts" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Due to brokers</link:label>
    <link:label id="lab_efc_IncreaseDecreaseInDueToBrokersMarginAccounts_label_en-US" xlink:label="lab_efc_IncreaseDecreaseInDueToBrokersMarginAccounts" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase Decrease In Due To Brokers Margin Accounts</link:label>
    <link:label id="lab_efc_IncreaseDecreaseInDueToBrokersMarginAccounts_documentation_en-US" xlink:label="lab_efc_IncreaseDecreaseInDueToBrokersMarginAccounts" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Carrying value of amounts transferred from third parties for security purposes that are expected to be returned or applied towards payment in the future.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_IncreaseDecreaseInDueToBrokersMarginAccounts" xlink:href="efc-20201231.xsd#efc_IncreaseDecreaseInDueToBrokersMarginAccounts"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_IncreaseDecreaseInDueToBrokersMarginAccounts" xlink:to="lab_efc_IncreaseDecreaseInDueToBrokersMarginAccounts" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CollateralOnRepurchaseAgreementsMember_d4a12d95-f815-442a-a763-9af300a71690_terseLabel_en-US" xlink:label="lab_efc_CollateralOnRepurchaseAgreementsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Collateral On Repurchase Agreements Held By Dealers [Member]</link:label>
    <link:label id="lab_efc_CollateralOnRepurchaseAgreementsMember_label_en-US" xlink:label="lab_efc_CollateralOnRepurchaseAgreementsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Collateral On Repurchase Agreements [Member]</link:label>
    <link:label id="lab_efc_CollateralOnRepurchaseAgreementsMember_documentation_en-US" xlink:label="lab_efc_CollateralOnRepurchaseAgreementsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Collateral On Repurchase Agreements [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CollateralOnRepurchaseAgreementsMember" xlink:href="efc-20201231.xsd#efc_CollateralOnRepurchaseAgreementsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CollateralOnRepurchaseAgreementsMember" xlink:to="lab_efc_CollateralOnRepurchaseAgreementsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CreditDefaultSwapBuyingProtectionMember_5f1af85d-ad64-4875-829f-fc70ef66c50a_terseLabel_en-US" xlink:label="lab_us-gaap_CreditDefaultSwapBuyingProtectionMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swap, Buying Protection [Member]</link:label>
    <link:label id="lab_us-gaap_CreditDefaultSwapBuyingProtectionMember_label_en-US" xlink:label="lab_us-gaap_CreditDefaultSwapBuyingProtectionMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swap, Buying Protection [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CreditDefaultSwapBuyingProtectionMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CreditDefaultSwapBuyingProtectionMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CreditDefaultSwapBuyingProtectionMember" xlink:to="lab_us-gaap_CreditDefaultSwapBuyingProtectionMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationExpenses_05fb8fa0-3d0d-4ae9-92c4-1028bbe9343a_terseLabel_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationExpenses" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Expenses</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationExpenses_label_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationExpenses" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Expenses</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationExpenses_documentation_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationExpenses" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount of expenses, other than interest expense, reported by equity method investee.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationExpenses" xlink:href="efc-20201231.xsd#efc_EquityMethodInvestmentSummarizedFinancialInformationExpenses"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationExpenses" xlink:to="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationExpenses" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InterestandDividendsReceivable_6afe0aa4-1c5f-4678-bd41-e98470c0de62_terseLabel_en-US" xlink:label="lab_efc_InterestandDividendsReceivable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest and Dividends Receivable</link:label>
    <link:label id="lab_efc_InterestandDividendsReceivable_label_en-US" xlink:label="lab_efc_InterestandDividendsReceivable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest and Dividends Receivable</link:label>
    <link:label id="lab_efc_InterestandDividendsReceivable_documentation_en-US" xlink:label="lab_efc_InterestandDividendsReceivable" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Sum of the carrying amount as of the balance sheet date of (a) interest earned but not received - also called accrued interest or accrued interest receivable, and (b) dividends declared but not received.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InterestandDividendsReceivable" xlink:href="efc-20201231.xsd#efc_InterestandDividendsReceivable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InterestandDividendsReceivable" xlink:to="lab_efc_InterestandDividendsReceivable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritizationTransactionsAbstract_ebbc262d-cd1a-46ea-aa5e-fdbbcf1e79c7_terseLabel_en-US" xlink:label="lab_efc_SecuritizationTransactionsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization Transactions [Abstract]</link:label>
    <link:label id="lab_efc_SecuritizationTransactionsAbstract_label_en-US" xlink:label="lab_efc_SecuritizationTransactionsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization Transactions [Abstract]</link:label>
    <link:label id="lab_efc_SecuritizationTransactionsAbstract_documentation_en-US" xlink:label="lab_efc_SecuritizationTransactionsAbstract" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization Transactions [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritizationTransactionsAbstract" xlink:href="efc-20201231.xsd#efc_SecuritizationTransactionsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritizationTransactionsAbstract" xlink:to="lab_efc_SecuritizationTransactionsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MeasurementInputDefaultRateMember_39f26b68-7977-48b6-99b5-93815ecead89_terseLabel_en-US" xlink:label="lab_us-gaap_MeasurementInputDefaultRateMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, Default Rate</link:label>
    <link:label id="lab_us-gaap_MeasurementInputDefaultRateMember_label_en-US" xlink:label="lab_us-gaap_MeasurementInputDefaultRateMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, Default Rate [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MeasurementInputDefaultRateMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MeasurementInputDefaultRateMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MeasurementInputDefaultRateMember" xlink:to="lab_us-gaap_MeasurementInputDefaultRateMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreements_78cd2cfa-ad63-4cfc-9541-90bba98301b5_terseLabel_en-US" xlink:label="lab_us-gaap_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instruments transferred as collateral for repurchase agreements</link:label>
    <link:label id="lab_us-gaap_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreements_label_en-US" xlink:label="lab_us-gaap_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Pledged Financial Instruments, Not Separately Reported, Securities for Repurchase Agreements</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreements" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreements"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreements" xlink:to="lab_us-gaap_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreements" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_BarclaysCapitalIncCollateralizedbyParValue1900ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member_311d23a7-c998-4a95-8202-8d43ecafe7d9_terseLabel_en-US" xlink:label="lab_efc_BarclaysCapitalIncCollateralizedbyParValue1900ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Barclays Capital Inc Collateralized by Par Value $1,900 Exchange-Traded Corporate Debt, Coupon 5.95%, Maturity Date 2026 12 [Member]</link:label>
    <link:label id="lab_efc_BarclaysCapitalIncCollateralizedbyParValue1900ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member_label_en-US" xlink:label="lab_efc_BarclaysCapitalIncCollateralizedbyParValue1900ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Barclays Capital Inc Collateralized by Par Value $1,900 Exchange-Traded Corporate Debt, Coupon 5.95%, Maturity Date 2026 12 [Member]</link:label>
    <link:label id="lab_efc_BarclaysCapitalIncCollateralizedbyParValue1900ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member_documentation_en-US" xlink:label="lab_efc_BarclaysCapitalIncCollateralizedbyParValue1900ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Barclays Capital Inc Collateralized by Par Value $1,900 Exchange-Traded Corporate Debt, Coupon 5.95%, Maturity Date 2026 12 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_BarclaysCapitalIncCollateralizedbyParValue1900ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:href="efc-20201231.xsd#efc_BarclaysCapitalIncCollateralizedbyParValue1900ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_BarclaysCapitalIncCollateralizedbyParValue1900ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:to="lab_efc_BarclaysCapitalIncCollateralizedbyParValue1900ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CashAndCashEquivalentsDisclosureTextBlock_16a2732c-a0a6-49fb-85ca-badf9b0be131_terseLabel_en-US" xlink:label="lab_us-gaap_CashAndCashEquivalentsDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Restricted Cash</link:label>
    <link:label id="lab_us-gaap_CashAndCashEquivalentsDisclosureTextBlock_label_en-US" xlink:label="lab_us-gaap_CashAndCashEquivalentsDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash and Cash Equivalents Disclosure [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CashAndCashEquivalentsDisclosureTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CashAndCashEquivalentsDisclosureTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CashAndCashEquivalentsDisclosureTextBlock" xlink:to="lab_us-gaap_CashAndCashEquivalentsDisclosureTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Interestincomerelatedtoconsolidatedsecuritizationtrust_0554458b-1837-4749-a661-9bbe47816ae1_terseLabel_en-US" xlink:label="lab_efc_Interestincomerelatedtoconsolidatedsecuritizationtrust" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest income related to consolidated securitization trust</link:label>
    <link:label id="lab_efc_Interestincomerelatedtoconsolidatedsecuritizationtrust_label_en-US" xlink:label="lab_efc_Interestincomerelatedtoconsolidatedsecuritizationtrust" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest income related to consolidated securitization trust</link:label>
    <link:label id="lab_efc_Interestincomerelatedtoconsolidatedsecuritizationtrust_documentation_en-US" xlink:label="lab_efc_Interestincomerelatedtoconsolidatedsecuritizationtrust" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest income related to consolidated securitization trust</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Interestincomerelatedtoconsolidatedsecuritizationtrust" xlink:href="efc-20201231.xsd#efc_Interestincomerelatedtoconsolidatedsecuritizationtrust"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Interestincomerelatedtoconsolidatedsecuritizationtrust" xlink:to="lab_efc_Interestincomerelatedtoconsolidatedsecuritizationtrust" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTable_b9a170e3-29fa-41a6-9d72-b47bbc371e8d_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Measurement Inputs and Valuation Techniques [Table]</link:label>
    <link:label id="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTable_label_en-US" xlink:label="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Measurement Inputs and Valuation Techniques [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTable" xlink:to="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AssetsAbstract_d824f973-676c-4a44-bf94-4004a62b5db7_terseLabel_en-US" xlink:label="lab_us-gaap_AssetsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets</link:label>
    <link:label id="lab_us-gaap_AssetsAbstract_label_en-US" xlink:label="lab_us-gaap_AssetsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AssetsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AssetsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AssetsAbstract" xlink:to="lab_us-gaap_AssetsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedGainBeforeTax_ab799783-a1c5-4873-bd72-2bd060c60eb7_terseLabel_en-US" xlink:label="lab_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedGainBeforeTax" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Gross Unrealized Gains</link:label>
    <link:label id="lab_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedGainBeforeTax_label_en-US" xlink:label="lab_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedGainBeforeTax" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Accumulated Gross Unrealized Gain, before Tax</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedGainBeforeTax" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedGainBeforeTax"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedGainBeforeTax" xlink:to="lab_us-gaap_AvailableForSaleDebtSecuritiesAccumulatedGrossUnrealizedGainBeforeTax" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AvailableForSaleSecuritiesGrossRealizedLosses_a18a1075-6433-4d8f-b0f6-868535dfcde5_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_AvailableForSaleSecuritiesGrossRealizedLosses" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Available-for-sale Securities, Gross Realized Losses</link:label>
    <link:label id="lab_us-gaap_AvailableForSaleSecuritiesGrossRealizedLosses_label_en-US" xlink:label="lab_us-gaap_AvailableForSaleSecuritiesGrossRealizedLosses" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Available-for-sale Securities, Gross Realized Losses</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AvailableForSaleSecuritiesGrossRealizedLosses" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AvailableForSaleSecuritiesGrossRealizedLosses"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AvailableForSaleSecuritiesGrossRealizedLosses" xlink:to="lab_us-gaap_AvailableForSaleSecuritiesGrossRealizedLosses" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999000Member_50f2e302-eeb9-47c3-8cc7-4aec52c30116_terseLabel_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999000Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $5,000,000 and $9,999,000</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999000Member_label_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999000Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $5,000,000 and $9,999,000 [Member]</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999000Member_documentation_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999000Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $5,000,000 and $9,999,000</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999000Member" xlink:href="efc-20201231.xsd#efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999000Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999000Member" xlink:to="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999000Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInNextRollingTwelveMonths_05ccf3fb-ed4d-4d75-b852-61c60924a7b6_terseLabel_en-US" xlink:label="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInNextRollingTwelveMonths" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Maturities, Repayments of Principal in Next Rolling Twelve Months</link:label>
    <link:label id="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInNextRollingTwelveMonths_label_en-US" xlink:label="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInNextRollingTwelveMonths" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Maturities, Repayments of Principal in Next Rolling Twelve Months</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInNextRollingTwelveMonths" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInNextRollingTwelveMonths"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInNextRollingTwelveMonths" xlink:to="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInNextRollingTwelveMonths" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_TotalOutstandingCollateral_3a5ab1a5-32c9-4bd6-a2de-0ec81c4dfdbf_terseLabel_en-US" xlink:label="lab_efc_TotalOutstandingCollateral" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total outstanding collateral</link:label>
    <link:label id="lab_efc_TotalOutstandingCollateral_label_en-US" xlink:label="lab_efc_TotalOutstandingCollateral" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Outstanding Collateral</link:label>
    <link:label id="lab_efc_TotalOutstandingCollateral_documentation_en-US" xlink:label="lab_efc_TotalOutstandingCollateral" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equals the underlying current outstanding collateral expressed as 100%.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_TotalOutstandingCollateral" xlink:href="efc-20201231.xsd#efc_TotalOutstandingCollateral"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_TotalOutstandingCollateral" xlink:to="lab_efc_TotalOutstandingCollateral" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtInstrumentUnamortizedDiscountPremiumNet_35edee57-568b-4366-bfef-6aa4d35b0e70_terseLabel_en-US" xlink:label="lab_us-gaap_DebtInstrumentUnamortizedDiscountPremiumNet" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unamortized Premium (Discount)</link:label>
    <link:label id="lab_us-gaap_DebtInstrumentUnamortizedDiscountPremiumNet_label_en-US" xlink:label="lab_us-gaap_DebtInstrumentUnamortizedDiscountPremiumNet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Instrument, Unamortized Discount (Premium), Net</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtInstrumentUnamortizedDiscountPremiumNet" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtInstrumentUnamortizedDiscountPremiumNet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtInstrumentUnamortizedDiscountPremiumNet" xlink:to="lab_us-gaap_DebtInstrumentUnamortizedDiscountPremiumNet" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CurrentIncomeTaxExpenseBenefit_2f28ae29-9365-4e66-8309-9f212eb66b7c_terseLabel_en-US" xlink:label="lab_us-gaap_CurrentIncomeTaxExpenseBenefit" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Current Income Tax Expense (Benefit)</link:label>
    <link:label id="lab_us-gaap_CurrentIncomeTaxExpenseBenefit_label_en-US" xlink:label="lab_us-gaap_CurrentIncomeTaxExpenseBenefit" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Current Income Tax Expense (Benefit)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CurrentIncomeTaxExpenseBenefit" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CurrentIncomeTaxExpenseBenefit"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CurrentIncomeTaxExpenseBenefit" xlink:to="lab_us-gaap_CurrentIncomeTaxExpenseBenefit" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncreaseDecreaseInDepositsOutstanding_152f996a-a2e0-42b8-855a-16aa922b92a6_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInDepositsOutstanding" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Due from brokers, net</link:label>
    <link:label id="lab_us-gaap_IncreaseDecreaseInDepositsOutstanding_label_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInDepositsOutstanding" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (Decrease) in Deposits Outstanding</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncreaseDecreaseInDepositsOutstanding" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncreaseDecreaseInDepositsOutstanding"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncreaseDecreaseInDepositsOutstanding" xlink:to="lab_us-gaap_IncreaseDecreaseInDepositsOutstanding" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ReverseMortgagesMember_710bc804-a891-4865-b284-290577114a23_terseLabel_en-US" xlink:label="lab_efc_ReverseMortgagesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reverse mortgages</link:label>
    <link:label id="lab_efc_ReverseMortgagesMember_label_en-US" xlink:label="lab_efc_ReverseMortgagesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reverse Mortgages [Member]</link:label>
    <link:label id="lab_efc_ReverseMortgagesMember_documentation_en-US" xlink:label="lab_efc_ReverseMortgagesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reverse Mortgages [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ReverseMortgagesMember" xlink:href="efc-20201231.xsd#efc_ReverseMortgagesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ReverseMortgagesMember" xlink:to="lab_efc_ReverseMortgagesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentOwnedAtCost_9c3992a2-96d7-443f-a42f-14146a9a2cb3_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentOwnedAtCost" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments, at fair value, Cost</link:label>
    <link:label id="lab_us-gaap_InvestmentOwnedAtCost_271c955f-4b34-4423-8eca-581b2f6706b6_totalLabel_en-US" xlink:label="lab_us-gaap_InvestmentOwnedAtCost" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amortized Cost</link:label>
    <link:label id="lab_us-gaap_InvestmentOwnedAtCost_label_en-US" xlink:label="lab_us-gaap_InvestmentOwnedAtCost" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Owned, at Cost</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentOwnedAtCost" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentOwnedAtCost"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentOwnedAtCost" xlink:to="lab_us-gaap_InvestmentOwnedAtCost" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PrincipalPaymentsOnOtherSecuredBorrowingsNonCash_3631a541-98b0-48eb-9493-8248a9427a15_terseLabel_en-US" xlink:label="lab_efc_PrincipalPaymentsOnOtherSecuredBorrowingsNonCash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal payments on Other secured borrowings (non-cash)</link:label>
    <link:label id="lab_efc_PrincipalPaymentsOnOtherSecuredBorrowingsNonCash_label_en-US" xlink:label="lab_efc_PrincipalPaymentsOnOtherSecuredBorrowingsNonCash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal payments on Other secured borrowings (non-cash)</link:label>
    <link:label id="lab_efc_PrincipalPaymentsOnOtherSecuredBorrowingsNonCash_documentation_en-US" xlink:label="lab_efc_PrincipalPaymentsOnOtherSecuredBorrowingsNonCash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal payments on Other secured borrowings (non-cash)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PrincipalPaymentsOnOtherSecuredBorrowingsNonCash" xlink:href="efc-20201231.xsd#efc_PrincipalPaymentsOnOtherSecuredBorrowingsNonCash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PrincipalPaymentsOnOtherSecuredBorrowingsNonCash" xlink:to="lab_efc_PrincipalPaymentsOnOtherSecuredBorrowingsNonCash" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonAgencyInterestOnlySecuritiesMember_3a6c7d8b-7831-4767-9776-b9210b881d19_terseLabel_en-US" xlink:label="lab_efc_NonAgencyInterestOnlySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Agency IOs</link:label>
    <link:label id="lab_efc_NonAgencyInterestOnlySecuritiesMember_label_en-US" xlink:label="lab_efc_NonAgencyInterestOnlySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Agency Interest Only Securities [Member]</link:label>
    <link:label id="lab_efc_NonAgencyInterestOnlySecuritiesMember_documentation_en-US" xlink:label="lab_efc_NonAgencyInterestOnlySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Agency Interest Only Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonAgencyInterestOnlySecuritiesMember" xlink:href="efc-20201231.xsd#efc_NonAgencyInterestOnlySecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonAgencyInterestOnlySecuritiesMember" xlink:to="lab_efc_NonAgencyInterestOnlySecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonExchangeTradedPreferredEquityInvestmentinConsumerLoanOriginatorsMemberDomain_206e2d0c-3745-4595-8f9f-666f1c61a4f5_terseLabel_en-US" xlink:label="lab_efc_NonExchangeTradedPreferredEquityInvestmentinConsumerLoanOriginatorsMemberDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Preferred Equity Investment in Consumer Loan Originators [Member] [Domain]</link:label>
    <link:label id="lab_efc_NonExchangeTradedPreferredEquityInvestmentinConsumerLoanOriginatorsMemberDomain_label_en-US" xlink:label="lab_efc_NonExchangeTradedPreferredEquityInvestmentinConsumerLoanOriginatorsMemberDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Preferred Equity Investment in Consumer Loan Originators [Member] [Domain]</link:label>
    <link:label id="lab_efc_NonExchangeTradedPreferredEquityInvestmentinConsumerLoanOriginatorsMemberDomain_documentation_en-US" xlink:label="lab_efc_NonExchangeTradedPreferredEquityInvestmentinConsumerLoanOriginatorsMemberDomain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Preferred Equity Investment in Consumer Loan Originators [Member] [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonExchangeTradedPreferredEquityInvestmentinConsumerLoanOriginatorsMemberDomain" xlink:href="efc-20201231.xsd#efc_NonExchangeTradedPreferredEquityInvestmentinConsumerLoanOriginatorsMemberDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonExchangeTradedPreferredEquityInvestmentinConsumerLoanOriginatorsMemberDomain" xlink:to="lab_efc_NonExchangeTradedPreferredEquityInvestmentinConsumerLoanOriginatorsMemberDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FixedRateResidentialMortgageMember_b0d077ef-87de-43e9-8587-755b6e69321d_terseLabel_en-US" xlink:label="lab_us-gaap_FixedRateResidentialMortgageMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed Rate Residential Mortgage</link:label>
    <link:label id="lab_us-gaap_FixedRateResidentialMortgageMember_label_en-US" xlink:label="lab_us-gaap_FixedRateResidentialMortgageMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed Rate Residential Mortgage [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FixedRateResidentialMortgageMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FixedRateResidentialMortgageMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FixedRateResidentialMortgageMember" xlink:to="lab_us-gaap_FixedRateResidentialMortgageMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LessthanthreeyearsMember_040f6647-71ec-420d-8599-79a3ab8b5c04_terseLabel_en-US" xlink:label="lab_efc_LessthanthreeyearsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Less than three years [Member]</link:label>
    <link:label id="lab_efc_LessthanthreeyearsMember_label_en-US" xlink:label="lab_efc_LessthanthreeyearsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Less than three years [Member]</link:label>
    <link:label id="lab_efc_LessthanthreeyearsMember_documentation_en-US" xlink:label="lab_efc_LessthanthreeyearsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Less than three years [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LessthanthreeyearsMember" xlink:href="efc-20201231.xsd#efc_LessthanthreeyearsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LessthanthreeyearsMember" xlink:to="lab_efc_LessthanthreeyearsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2040Member_d87afbb9-830c-4b3a-9dc8-80b641ea7a0e_terseLabel_en-US" xlink:label="lab_efc_A2040Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2040</link:label>
    <link:label id="lab_efc_A2040Member_label_en-US" xlink:label="lab_efc_A2040Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2040 [Member]</link:label>
    <link:label id="lab_efc_A2040Member_documentation_en-US" xlink:label="lab_efc_A2040Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2040</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2040Member" xlink:href="efc-20201231.xsd#efc_A2040Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2040Member" xlink:to="lab_efc_A2040Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeFinancialInstrumentsAssetsMember_3060f1b0-6b37-4f5b-a94f-da70cdcca9ac_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeFinancialInstrumentsAssetsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Derivatives - Assets</link:label>
    <link:label id="lab_us-gaap_DerivativeFinancialInstrumentsAssetsMember_label_en-US" xlink:label="lab_us-gaap_DerivativeFinancialInstrumentsAssetsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Financial Instruments, Assets [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeFinancialInstrumentsAssetsMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeFinancialInstrumentsAssetsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeFinancialInstrumentsAssetsMember" xlink:to="lab_us-gaap_DerivativeFinancialInstrumentsAssetsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityWellKnownSeasonedIssuer_31311200-897f-44f7-9b2f-51158a4ed784_terseLabel_en-US" xlink:label="lab_dei_EntityWellKnownSeasonedIssuer" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Well-known Seasoned Issuer</link:label>
    <link:label id="lab_dei_EntityWellKnownSeasonedIssuer_label_en-US" xlink:label="lab_dei_EntityWellKnownSeasonedIssuer" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Well-known Seasoned Issuer</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityWellKnownSeasonedIssuer" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityWellKnownSeasonedIssuer"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityWellKnownSeasonedIssuer" xlink:to="lab_dei_EntityWellKnownSeasonedIssuer" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_EuropeMember_cca1899b-8715-4770-9bdd-4a1f72b78f8c_terseLabel_en-US" xlink:label="lab_srt_EuropeMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Europe [Member]</link:label>
    <link:label id="lab_srt_EuropeMember_label_en-US" xlink:label="lab_srt_EuropeMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Europe [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_EuropeMember" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_EuropeMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_EuropeMember" xlink:to="lab_srt_EuropeMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAbstract_0e9d0795-f1de-4c86-b594-1be9374a651c_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase agreements</link:label>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAbstract_label_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Sold under Agreements to Repurchase [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAbstract" xlink:to="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueByMeasurementBasisAxis_3d391842-016e-4a56-9c5a-5f95c7ba595c_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueByMeasurementBasisAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Basis [Axis]</link:label>
    <link:label id="lab_us-gaap_FairValueByMeasurementBasisAxis_label_en-US" xlink:label="lab_us-gaap_FairValueByMeasurementBasisAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Basis [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueByMeasurementBasisAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueByMeasurementBasisAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueByMeasurementBasisAxis" xlink:to="lab_us-gaap_FairValueByMeasurementBasisAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RevenueRecognitionInterest_6ed557d0-85a2-48f7-b4a5-3623c68be11b_terseLabel_en-US" xlink:label="lab_us-gaap_RevenueRecognitionInterest" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Income</link:label>
    <link:label id="lab_us-gaap_RevenueRecognitionInterest_label_en-US" xlink:label="lab_us-gaap_RevenueRecognitionInterest" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Revenue Recognition, Interest [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RevenueRecognitionInterest" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RevenueRecognitionInterest"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RevenueRecognitionInterest" xlink:to="lab_us-gaap_RevenueRecognitionInterest" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StockIssuedDuringPeriodValueConversionOfUnits_d597962b-372d-43d6-b8d3-1212e08b0513_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_StockIssuedDuringPeriodValueConversionOfUnits" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stock Issued During Period, Value, Conversion of Units</link:label>
    <link:label id="lab_us-gaap_StockIssuedDuringPeriodValueConversionOfUnits_label_en-US" xlink:label="lab_us-gaap_StockIssuedDuringPeriodValueConversionOfUnits" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stock Issued During Period, Value, Conversion of Units</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StockIssuedDuringPeriodValueConversionOfUnits" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockIssuedDuringPeriodValueConversionOfUnits"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StockIssuedDuringPeriodValueConversionOfUnits" xlink:to="lab_us-gaap_StockIssuedDuringPeriodValueConversionOfUnits" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansOnRealEstateLoanTypeDomain_b60fba99-0998-4c0d-9c43-09378a41c0cc_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateLoanTypeDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Loan Type [Domain]</link:label>
    <link:label id="lab_srt_MortgageLoansOnRealEstateLoanTypeDomain_label_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateLoanTypeDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Loan Type [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansOnRealEstateLoanTypeDomain" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansOnRealEstateLoanTypeDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansOnRealEstateLoanTypeDomain" xlink:to="lab_srt_MortgageLoansOnRealEstateLoanTypeDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeRemainingMaturity1_3dc727f1-ca7d-4aac-a994-87788d9772dd_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeRemainingMaturity1" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Remaining Years to Maturity</link:label>
    <link:label id="lab_us-gaap_DerivativeRemainingMaturity1_label_en-US" xlink:label="lab_us-gaap_DerivativeRemainingMaturity1" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative, Remaining Maturity</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeRemainingMaturity1" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeRemainingMaturity1"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeRemainingMaturity1" xlink:to="lab_us-gaap_DerivativeRemainingMaturity1" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1750000And1999999Member_8112de67-cbaf-4c51-8569-4d6c6e2b3c70_terseLabel_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1750000And1999999Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $1,750,000 and $1,999,999</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1750000And1999999Member_label_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1750000And1999999Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $1,750,000 and $1,999,999 [Member]</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1750000And1999999Member_documentation_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1750000And1999999Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $1,750,000 and $1,999,999</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1750000And1999999Member" xlink:href="efc-20201231.xsd#efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1750000And1999999Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1750000And1999999Member" xlink:to="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1750000And1999999Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardOptionsGrantsinPeriodUnvested_29693ef9-d9f9-4162-af08-2f6531b08fd5_terseLabel_en-US" xlink:label="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardOptionsGrantsinPeriodUnvested" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Grants in Period (in shares)</link:label>
    <link:label id="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardOptionsGrantsinPeriodUnvested_908da8cd-d95c-4325-8b65-87454e1d2c43_verboseLabel_en-US" xlink:label="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardOptionsGrantsinPeriodUnvested" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of OP LTIP Units (in units)</link:label>
    <link:label id="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardOptionsGrantsinPeriodUnvested_label_en-US" xlink:label="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardOptionsGrantsinPeriodUnvested" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Compensation Arrangement by Share-based Payment Award, Options, Grants in Period, Unvested</link:label>
    <link:label id="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardOptionsGrantsinPeriodUnvested_documentation_en-US" xlink:label="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardOptionsGrantsinPeriodUnvested" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Compensation Arrangement by Share-based Payment Award, Options, Grants in Period, Unvested</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardOptionsGrantsinPeriodUnvested" xlink:href="efc-20201231.xsd#efc_SharebasedCompensationArrangementbySharebasedPaymentAwardOptionsGrantsinPeriodUnvested"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardOptionsGrantsinPeriodUnvested" xlink:to="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardOptionsGrantsinPeriodUnvested" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritiesCollateralizedByParValue_b44d38d5-4ebc-4411-b540-358b4c9a770b_terseLabel_en-US" xlink:label="lab_efc_SecuritiesCollateralizedByParValue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Collateralized By Par Value</link:label>
    <link:label id="lab_efc_SecuritiesCollateralizedByParValue_label_en-US" xlink:label="lab_efc_SecuritiesCollateralizedByParValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Collateralized By Par Value</link:label>
    <link:label id="lab_efc_SecuritiesCollateralizedByParValue_documentation_en-US" xlink:label="lab_efc_SecuritiesCollateralizedByParValue" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Par value of collateral underlying repurchase agreement.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritiesCollateralizedByParValue" xlink:href="efc-20201231.xsd#efc_SecuritiesCollateralizedByParValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritiesCollateralizedByParValue" xlink:to="lab_efc_SecuritiesCollateralizedByParValue" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfCashAndCashEquivalentsTableTextBlock_9ef58390-fb05-43ec-8600-9918bac1527a_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfCashAndCashEquivalentsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Cash and Cash Equivalents</link:label>
    <link:label id="lab_us-gaap_ScheduleOfCashAndCashEquivalentsTableTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfCashAndCashEquivalentsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Cash and Cash Equivalents [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfCashAndCashEquivalentsTableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfCashAndCashEquivalentsTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfCashAndCashEquivalentsTableTextBlock" xlink:to="lab_us-gaap_ScheduleOfCashAndCashEquivalentsTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeInstrumentRiskAxis_d2912f3e-70cb-4603-afb6-6c199876292b_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeInstrumentRiskAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Instrument [Axis]</link:label>
    <link:label id="lab_us-gaap_DerivativeInstrumentRiskAxis_label_en-US" xlink:label="lab_us-gaap_DerivativeInstrumentRiskAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Instrument [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeInstrumentRiskAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeInstrumentRiskAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeInstrumentRiskAxis" xlink:to="lab_us-gaap_DerivativeInstrumentRiskAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncreaseDecreaseInStockholdersEquityRollForward_7f06917d-8aef-476b-bb19-0015be32ef7e_verboseLabel_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInStockholdersEquityRollForward" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CHANGE IN SHAREHOLDERS&#8217; EQUITY RESULTING FROM OPERATIONS</link:label>
    <link:label id="lab_us-gaap_IncreaseDecreaseInStockholdersEquityRollForward_5fc1ffbf-8787-45fc-8343-de1ff23d7484_terseLabel_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInStockholdersEquityRollForward" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (Decrease) in Stockholders' Equity [Roll Forward]</link:label>
    <link:label id="lab_us-gaap_IncreaseDecreaseInStockholdersEquityRollForward_label_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInStockholdersEquityRollForward" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (Decrease) in Stockholders' Equity [Roll Forward]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncreaseDecreaseInStockholdersEquityRollForward" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncreaseDecreaseInStockholdersEquityRollForward"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncreaseDecreaseInStockholdersEquityRollForward" xlink:to="lab_us-gaap_IncreaseDecreaseInStockholdersEquityRollForward" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Investmentyield_0757e469-84bd-45e2-b0b5-ec4e9da28ead_terseLabel_en-US" xlink:label="lab_efc_Investmentyield" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Weighted Average Yield</link:label>
    <link:label id="lab_efc_Investmentyield_label_en-US" xlink:label="lab_efc_Investmentyield" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment yield</link:label>
    <link:label id="lab_efc_Investmentyield_documentation_en-US" xlink:label="lab_efc_Investmentyield" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment yield</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Investmentyield" xlink:href="efc-20201231.xsd#efc_Investmentyield"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Investmentyield" xlink:to="lab_efc_Investmentyield" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancialAssetsSoldUnderAgreementToRepurchaseMember_9a6103f5-0ea2-4e42-b0da-635fce84ec41_terseLabel_en-US" xlink:label="lab_us-gaap_FinancialAssetsSoldUnderAgreementToRepurchaseMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Assets Sold under Agreement to Repurchase [Member]</link:label>
    <link:label id="lab_us-gaap_FinancialAssetsSoldUnderAgreementToRepurchaseMember_label_en-US" xlink:label="lab_us-gaap_FinancialAssetsSoldUnderAgreementToRepurchaseMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Assets Sold under Agreement to Repurchase [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancialAssetsSoldUnderAgreementToRepurchaseMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancialAssetsSoldUnderAgreementToRepurchaseMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancialAssetsSoldUnderAgreementToRepurchaseMember" xlink:to="lab_us-gaap_FinancialAssetsSoldUnderAgreementToRepurchaseMember" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_RangeAxis_2b60ab1a-031d-436f-a0f0-525853eae111_terseLabel_en-US" xlink:label="lab_srt_RangeAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Statistical Measurement [Axis]</link:label>
    <link:label id="lab_srt_RangeAxis_label_en-US" xlink:label="lab_srt_RangeAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Statistical Measurement [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_RangeAxis" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_RangeAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_RangeAxis" xlink:to="lab_srt_RangeAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SupplementalCashFlowInformationAbstract_1a5dedc4-d973-4e42-b4ae-3f6a0a916600_terseLabel_en-US" xlink:label="lab_us-gaap_SupplementalCashFlowInformationAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Supplemental disclosure of cash flow information:</link:label>
    <link:label id="lab_us-gaap_SupplementalCashFlowInformationAbstract_label_en-US" xlink:label="lab_us-gaap_SupplementalCashFlowInformationAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Supplemental Cash Flow Information [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SupplementalCashFlowInformationAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SupplementalCashFlowInformationAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SupplementalCashFlowInformationAbstract" xlink:to="lab_us-gaap_SupplementalCashFlowInformationAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_VestDecember112019Member_f9ea7516-8e2a-460d-8632-3f14587ca468_terseLabel_en-US" xlink:label="lab_efc_VestDecember112019Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 11, 2019 [Member]</link:label>
    <link:label id="lab_efc_VestDecember112019Member_label_en-US" xlink:label="lab_efc_VestDecember112019Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 11, 2019 [Member]</link:label>
    <link:label id="lab_efc_VestDecember112019Member_documentation_en-US" xlink:label="lab_efc_VestDecember112019Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 11, 2019 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_VestDecember112019Member" xlink:href="efc-20201231.xsd#efc_VestDecember112019Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_VestDecember112019Member" xlink:to="lab_efc_VestDecember112019Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CLOIISecuritizationMember_16a47814-d6c2-44e6-a8a6-baeae0463bbd_terseLabel_en-US" xlink:label="lab_efc_CLOIISecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLO II Securitization [Member]</link:label>
    <link:label id="lab_efc_CLOIISecuritizationMember_label_en-US" xlink:label="lab_efc_CLOIISecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLO II Securitization [Member]</link:label>
    <link:label id="lab_efc_CLOIISecuritizationMember_documentation_en-US" xlink:label="lab_efc_CLOIISecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLO II Securitization (CLO transaction priced December 2017 closed January 2018)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CLOIISecuritizationMember" xlink:href="efc-20201231.xsd#efc_CLOIISecuritizationMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CLOIISecuritizationMember" xlink:to="lab_efc_CLOIISecuritizationMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DistributedfromRiskRetentionVehicleMember_302689b4-814c-483e-9f39-ad9700694f94_terseLabel_en-US" xlink:label="lab_efc_DistributedfromRiskRetentionVehicleMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Distributed from Risk Retention Vehicle [Member]</link:label>
    <link:label id="lab_efc_DistributedfromRiskRetentionVehicleMember_label_en-US" xlink:label="lab_efc_DistributedfromRiskRetentionVehicleMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Distributed from Risk Retention Vehicle [Member]</link:label>
    <link:label id="lab_efc_DistributedfromRiskRetentionVehicleMember_documentation_en-US" xlink:label="lab_efc_DistributedfromRiskRetentionVehicleMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Distributed from Risk Retention Vehicle [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DistributedfromRiskRetentionVehicleMember" xlink:href="efc-20201231.xsd#efc_DistributedfromRiskRetentionVehicleMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DistributedfromRiskRetentionVehicleMember" xlink:to="lab_efc_DistributedfromRiskRetentionVehicleMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_AdjustableRateCommercialMortgageMember_68f52d61-b407-44ba-a890-b63614d78532_terseLabel_en-US" xlink:label="lab_efc_AdjustableRateCommercialMortgageMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Adjustable Rate Commercial Mortgage</link:label>
    <link:label id="lab_efc_AdjustableRateCommercialMortgageMember_label_en-US" xlink:label="lab_efc_AdjustableRateCommercialMortgageMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Adjustable Rate Commercial Mortgage [Member]</link:label>
    <link:label id="lab_efc_AdjustableRateCommercialMortgageMember_documentation_en-US" xlink:label="lab_efc_AdjustableRateCommercialMortgageMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Adjustable Rate Commercial Mortgage</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_AdjustableRateCommercialMortgageMember" xlink:href="efc-20201231.xsd#efc_AdjustableRateCommercialMortgageMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_AdjustableRateCommercialMortgageMember" xlink:to="lab_efc_AdjustableRateCommercialMortgageMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_WeightedAverageShareImpactOnDividendsPaid_c867719e-9ba8-42b8-b1be-bd4b054e6bc6_terseLabel_en-US" xlink:label="lab_efc_WeightedAverageShareImpactOnDividendsPaid" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Weighted Average Share Impact on Dividends Paid</link:label>
    <link:label id="lab_efc_WeightedAverageShareImpactOnDividendsPaid_label_en-US" xlink:label="lab_efc_WeightedAverageShareImpactOnDividendsPaid" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Weighted Average Share Impact on Dividends Paid</link:label>
    <link:label id="lab_efc_WeightedAverageShareImpactOnDividendsPaid_documentation_en-US" xlink:label="lab_efc_WeightedAverageShareImpactOnDividendsPaid" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Per share impact on dividend amount paid relating to shares issued/repurchased during the period as well as dividends paid to LTIP unit holders.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_WeightedAverageShareImpactOnDividendsPaid" xlink:href="efc-20201231.xsd#efc_WeightedAverageShareImpactOnDividendsPaid"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_WeightedAverageShareImpactOnDividendsPaid" xlink:to="lab_efc_WeightedAverageShareImpactOnDividendsPaid" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger_7de2eaf5-a974-4e18-ad25-b9e1bdd34817_terseLabel_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Greater than 12 Months</link:label>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger_label_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Continuous Unrealized Loss Position, 12 Months or Longer</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger" xlink:to="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLonger" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DividendsPreferredStockCash_f53b5e9f-bd6b-4562-8b05-3ff98d2236b5_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_DividendsPreferredStockCash" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends, Preferred Stock, Cash</link:label>
    <link:label id="lab_us-gaap_DividendsPreferredStockCash_label_en-US" xlink:label="lab_us-gaap_DividendsPreferredStockCash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends, Preferred Stock, Cash</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DividendsPreferredStockCash" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DividendsPreferredStockCash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DividendsPreferredStockCash" xlink:to="lab_us-gaap_DividendsPreferredStockCash" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_BasisOfAccountingPolicyPolicyTextBlock_5895e8d4-77ed-42cb-89de-118927c733cd_verboseLabel_en-US" xlink:label="lab_us-gaap_BasisOfAccountingPolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Basis of Presentation</link:label>
    <link:label id="lab_us-gaap_BasisOfAccountingPolicyPolicyTextBlock_label_en-US" xlink:label="lab_us-gaap_BasisOfAccountingPolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Basis of Accounting, Policy [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_BasisOfAccountingPolicyPolicyTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_BasisOfAccountingPolicyPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_BasisOfAccountingPolicyPolicyTextBlock" xlink:to="lab_us-gaap_BasisOfAccountingPolicyPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_BasicMaterialsMember_f5f6ead7-17c2-4b5d-a6d4-b356b09c7c0e_terseLabel_en-US" xlink:label="lab_efc_BasicMaterialsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Basic Materials [Member]</link:label>
    <link:label id="lab_efc_BasicMaterialsMember_label_en-US" xlink:label="lab_efc_BasicMaterialsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Basic Materials [Member]</link:label>
    <link:label id="lab_efc_BasicMaterialsMember_documentation_en-US" xlink:label="lab_efc_BasicMaterialsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Basic Materials [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_BasicMaterialsMember" xlink:href="efc-20201231.xsd#efc_BasicMaterialsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_BasicMaterialsMember" xlink:to="lab_efc_BasicMaterialsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Totalpercentageofdividendswithtaxcharacteristic_b6c5b52e-5249-4700-aa26-d38b63ac08c8_terseLabel_en-US" xlink:label="lab_efc_Totalpercentageofdividendswithtaxcharacteristic" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total, percentage of dividends with tax characteristic</link:label>
    <link:label id="lab_efc_Totalpercentageofdividendswithtaxcharacteristic_label_en-US" xlink:label="lab_efc_Totalpercentageofdividendswithtaxcharacteristic" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total, percentage of dividends with tax characteristic</link:label>
    <link:label id="lab_efc_Totalpercentageofdividendswithtaxcharacteristic_documentation_en-US" xlink:label="lab_efc_Totalpercentageofdividendswithtaxcharacteristic" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total, percentage of dividends with tax characteristic</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Totalpercentageofdividendswithtaxcharacteristic" xlink:href="efc-20201231.xsd#efc_Totalpercentageofdividendswithtaxcharacteristic"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Totalpercentageofdividendswithtaxcharacteristic" xlink:to="lab_efc_Totalpercentageofdividendswithtaxcharacteristic" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2024Member_3bcfabb7-c1fa-46e8-af20-ef651aca5ec8_terseLabel_en-US" xlink:label="lab_efc_A2024Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2024 [Member]</link:label>
    <link:label id="lab_efc_A2024Member_label_en-US" xlink:label="lab_efc_A2024Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2024 [Member]</link:label>
    <link:label id="lab_efc_A2024Member_documentation_en-US" xlink:label="lab_efc_A2024Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2024 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2024Member" xlink:href="efc-20201231.xsd#efc_A2024Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2024Member" xlink:to="lab_efc_A2024Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNetAvailableforsaleSecurities_d771270c-4481-4fb8-ba43-22f0a24e2282_terseLabel_en-US" xlink:label="lab_us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNetAvailableforsaleSecurities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">OTTI charge on securities</link:label>
    <link:label id="lab_us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNetAvailableforsaleSecurities_label_en-US" xlink:label="lab_us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNetAvailableforsaleSecurities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other than Temporary Impairment Losses, Investments, Portion Recognized in Earnings, Net, Available-for-sale Securities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNetAvailableforsaleSecurities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNetAvailableforsaleSecurities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNetAvailableforsaleSecurities" xlink:to="lab_us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNetAvailableforsaleSecurities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PrincipalAndInterestPrivateLabelSecuritiesMember_ac8128d7-2b7a-4d08-9ac5-942ad11a1dbd_terseLabel_en-US" xlink:label="lab_efc_PrincipalAndInterestPrivateLabelSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal And Interest Private Label Securities [Member]</link:label>
    <link:label id="lab_efc_PrincipalAndInterestPrivateLabelSecuritiesMember_label_en-US" xlink:label="lab_efc_PrincipalAndInterestPrivateLabelSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal And Interest Private Label Securities [Member]</link:label>
    <link:label id="lab_efc_PrincipalAndInterestPrivateLabelSecuritiesMember_documentation_en-US" xlink:label="lab_efc_PrincipalAndInterestPrivateLabelSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal and interest private label securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PrincipalAndInterestPrivateLabelSecuritiesMember" xlink:href="efc-20201231.xsd#efc_PrincipalAndInterestPrivateLabelSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PrincipalAndInterestPrivateLabelSecuritiesMember" xlink:to="lab_efc_PrincipalAndInterestPrivateLabelSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeNotionalAmount_0a27af96-f6bc-4769-b6ba-f16af83808f7_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeNotionalAmount" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Notional Amount</link:label>
    <link:label id="lab_us-gaap_DerivativeNotionalAmount_ab88dcd9-96b3-4651-91be-87cabcf08649_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_DerivativeNotionalAmount" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Notional Amount</link:label>
    <link:label id="lab_us-gaap_DerivativeNotionalAmount_6f6ae048-326d-4fe8-b8f2-921f7e1caab7_verboseLabel_en-US" xlink:label="lab_us-gaap_DerivativeNotionalAmount" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Notional value</link:label>
    <link:label id="lab_us-gaap_DerivativeNotionalAmount_label_en-US" xlink:label="lab_us-gaap_DerivativeNotionalAmount" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative, Notional Amount</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeNotionalAmount" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeNotionalAmount"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeNotionalAmount" xlink:to="lab_us-gaap_DerivativeNotionalAmount" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LongTermIncentivePlanUnitsMember_2c7dbabf-960a-4ced-99aa-bc24131e3ad1_terseLabel_en-US" xlink:label="lab_efc_LongTermIncentivePlanUnitsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-Term Incentive Plan Units [Member]</link:label>
    <link:label id="lab_efc_LongTermIncentivePlanUnitsMember_label_en-US" xlink:label="lab_efc_LongTermIncentivePlanUnitsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Term Incentive Plan Units [Member]</link:label>
    <link:label id="lab_efc_LongTermIncentivePlanUnitsMember_documentation_en-US" xlink:label="lab_efc_LongTermIncentivePlanUnitsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Term Incentive Plan Units [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LongTermIncentivePlanUnitsMember" xlink:href="efc-20201231.xsd#efc_LongTermIncentivePlanUnitsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LongTermIncentivePlanUnitsMember" xlink:to="lab_efc_LongTermIncentivePlanUnitsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ManagementFeeExpense_c67bde2d-2fa5-4da4-afcd-baad2ed28df3_terseLabel_en-US" xlink:label="lab_us-gaap_ManagementFeeExpense" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Base management fee to affiliate (Net of fee rebates)</link:label>
    <link:label id="lab_us-gaap_ManagementFeeExpense_4cb22d05-bf11-4a8c-9144-410947bf30e9_verboseLabel_en-US" xlink:label="lab_us-gaap_ManagementFeeExpense" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Base management fee to affiliate</link:label>
    <link:label id="lab_us-gaap_ManagementFeeExpense_label_en-US" xlink:label="lab_us-gaap_ManagementFeeExpense" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Management Fee Expense</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ManagementFeeExpense" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ManagementFeeExpense"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ManagementFeeExpense" xlink:to="lab_us-gaap_ManagementFeeExpense" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestexpense_2f6dca55-efb2-46d0-a60f-ae964e44914e_terseLabel_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestexpense" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Interest expense</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestexpense_label_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestexpense" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Interest expense</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestexpense_documentation_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestexpense" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount of interest expense reported by equity method investee.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestexpense" xlink:href="efc-20201231.xsd#efc_EquityMethodInvestmentSummarizedFinancialInformationInterestexpense"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestexpense" xlink:to="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestexpense" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EarningsPerShareBasicAbstract_df030599-adc8-44d1-ac05-87a8a023a3e9_terseLabel_en-US" xlink:label="lab_us-gaap_EarningsPerShareBasicAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Basic earnings per share of common stock:</link:label>
    <link:label id="lab_us-gaap_EarningsPerShareBasicAbstract_label_en-US" xlink:label="lab_us-gaap_EarningsPerShareBasicAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share, Basic [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EarningsPerShareBasicAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EarningsPerShareBasicAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EarningsPerShareBasicAbstract" xlink:to="lab_us-gaap_EarningsPerShareBasicAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EarningsPerShareBasicDistributed_e391a9c2-7381-493e-bfdf-9cb8eba4e4c5_terseLabel_en-US" xlink:label="lab_us-gaap_EarningsPerShareBasicDistributed" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Distributed (in usd per share)</link:label>
    <link:label id="lab_us-gaap_EarningsPerShareBasicDistributed_label_en-US" xlink:label="lab_us-gaap_EarningsPerShareBasicDistributed" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share, Basic, Distributed</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EarningsPerShareBasicDistributed" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EarningsPerShareBasicDistributed"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EarningsPerShareBasicDistributed" xlink:to="lab_us-gaap_EarningsPerShareBasicDistributed" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritiesAndLoansRealizedGainsLosses_95bf7e38-6101-4209-bc47-22d5f728fe0c_terseLabel_en-US" xlink:label="lab_efc_SecuritiesAndLoansRealizedGainsLosses" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized gains (losses) on securities and loans, net</link:label>
    <link:label id="lab_efc_SecuritiesAndLoansRealizedGainsLosses_4ee06dd2-6298-4b1e-87ca-533e543aa0d1_negatedTerseLabel_en-US" xlink:label="lab_efc_SecuritiesAndLoansRealizedGainsLosses" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized (gains) losses on securities and loans, net</link:label>
    <link:label id="lab_efc_SecuritiesAndLoansRealizedGainsLosses_label_en-US" xlink:label="lab_efc_SecuritiesAndLoansRealizedGainsLosses" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities And Loans, Realized Gains (Losses)</link:label>
    <link:label id="lab_efc_SecuritiesAndLoansRealizedGainsLosses_documentation_en-US" xlink:label="lab_efc_SecuritiesAndLoansRealizedGainsLosses" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities And Loans, Realized Gains (Losses)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritiesAndLoansRealizedGainsLosses" xlink:href="efc-20201231.xsd#efc_SecuritiesAndLoansRealizedGainsLosses"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritiesAndLoansRealizedGainsLosses" xlink:to="lab_efc_SecuritiesAndLoansRealizedGainsLosses" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporation30Year3Point50PercentMember_7294ccee-0cb3-446d-8e61-1c05aa547078_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporation30Year3Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation (30 Year) 3 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporation30Year3Point50PercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporation30Year3Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation (30 Year) 3 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporation30Year3Point50PercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporation30Year3Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation (30 Year) 3.50%</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporation30Year3Point50PercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporation30Year3Point50PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporation30Year3Point50PercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporation30Year3Point50PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtDisclosureTextBlock_60b984f8-55f7-44f5-87a2-93d23cc3aef5_terseLabel_en-US" xlink:label="lab_us-gaap_DebtDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Borrowings</link:label>
    <link:label id="lab_us-gaap_DebtDisclosureTextBlock_label_en-US" xlink:label="lab_us-gaap_DebtDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Disclosure [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtDisclosureTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtDisclosureTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtDisclosureTextBlock" xlink:to="lab_us-gaap_DebtDisclosureTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleOfNetInvestmentIncomeRatioToAverageEquityTableTextBlock_911a6f61-6ca3-4927-b382-b3073f38f985_terseLabel_en-US" xlink:label="lab_efc_ScheduleOfNetInvestmentIncomeRatioToAverageEquityTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Net Investment Income Ratio To Average Equity</link:label>
    <link:label id="lab_efc_ScheduleOfNetInvestmentIncomeRatioToAverageEquityTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleOfNetInvestmentIncomeRatioToAverageEquityTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Net Investment Income Ratio To Average Equity [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleOfNetInvestmentIncomeRatioToAverageEquityTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleOfNetInvestmentIncomeRatioToAverageEquityTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ratio of net investment income to average total equity (including non-controlling interest).</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleOfNetInvestmentIncomeRatioToAverageEquityTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleOfNetInvestmentIncomeRatioToAverageEquityTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleOfNetInvestmentIncomeRatioToAverageEquityTableTextBlock" xlink:to="lab_efc_ScheduleOfNetInvestmentIncomeRatioToAverageEquityTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeFixedInterestRate_19aa2a4c-e2e6-41b8-ad0b-ba32a915597a_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeFixedInterestRate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Pay Rate</link:label>
    <link:label id="lab_us-gaap_DerivativeFixedInterestRate_label_en-US" xlink:label="lab_us-gaap_DerivativeFixedInterestRate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative, Fixed Interest Rate</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeFixedInterestRate" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeFixedInterestRate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeFixedInterestRate" xlink:to="lab_us-gaap_DerivativeFixedInterestRate" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancingReceivableRecordedInvestmentPastDueLineItems_52a05122-dc8c-43e5-8c99-347b2a4208eb_terseLabel_en-US" xlink:label="lab_us-gaap_FinancingReceivableRecordedInvestmentPastDueLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable, Past Due [Line Items]</link:label>
    <link:label id="lab_us-gaap_FinancingReceivableRecordedInvestmentPastDueLineItems_label_en-US" xlink:label="lab_us-gaap_FinancingReceivableRecordedInvestmentPastDueLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable, Past Due [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancingReceivableRecordedInvestmentPastDueLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancingReceivableRecordedInvestmentPastDueLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancingReceivableRecordedInvestmentPastDueLineItems" xlink:to="lab_us-gaap_FinancingReceivableRecordedInvestmentPastDueLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentCompanyRealizedAndUnrealizedGainLossOnInvestmentAndForeignCurrency_aadb2a4a-4fd6-4c18-be1a-965a8dfa405e_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentCompanyRealizedAndUnrealizedGainLossOnInvestmentAndForeignCurrency" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Realized and Unrealized Gain (Loss) on Investment and Foreign Currency</link:label>
    <link:label id="lab_us-gaap_InvestmentCompanyRealizedAndUnrealizedGainLossOnInvestmentAndForeignCurrency_label_en-US" xlink:label="lab_us-gaap_InvestmentCompanyRealizedAndUnrealizedGainLossOnInvestmentAndForeignCurrency" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Realized and Unrealized Gain (Loss) on Investment and Foreign Currency</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentCompanyRealizedAndUnrealizedGainLossOnInvestmentAndForeignCurrency" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentCompanyRealizedAndUnrealizedGainLossOnInvestmentAndForeignCurrency"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentCompanyRealizedAndUnrealizedGainLossOnInvestmentAndForeignCurrency" xlink:to="lab_us-gaap_InvestmentCompanyRealizedAndUnrealizedGainLossOnInvestmentAndForeignCurrency" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CommonStockSharesAuthorized_fec1641b-b719-448b-a902-249617981041_terseLabel_en-US" xlink:label="lab_us-gaap_CommonStockSharesAuthorized" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common stock, shares authorized</link:label>
    <link:label id="lab_us-gaap_CommonStockSharesAuthorized_f7acc132-ced2-4461-a94b-614a23bcfde8_verboseLabel_en-US" xlink:label="lab_us-gaap_CommonStockSharesAuthorized" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common stock, shares authorized (in shares)</link:label>
    <link:label id="lab_us-gaap_CommonStockSharesAuthorized_label_en-US" xlink:label="lab_us-gaap_CommonStockSharesAuthorized" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common Stock, Shares Authorized</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CommonStockSharesAuthorized" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CommonStockSharesAuthorized"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CommonStockSharesAuthorized" xlink:to="lab_us-gaap_CommonStockSharesAuthorized" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2029Member_4ddc697a-afa6-46a8-85a4-2cef185f531d_terseLabel_en-US" xlink:label="lab_efc_A2029Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2029 [Member]</link:label>
    <link:label id="lab_efc_A2029Member_label_en-US" xlink:label="lab_efc_A2029Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2029 [Member]</link:label>
    <link:label id="lab_efc_A2029Member_documentation_en-US" xlink:label="lab_efc_A2029Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2029 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2029Member" xlink:href="efc-20201231.xsd#efc_A2029Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2029Member" xlink:to="lab_efc_A2029Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Excesscashcollateral_fe77ef28-dce4-438c-b32d-ccfb2ab7c1f5_terseLabel_en-US" xlink:label="lab_efc_Excesscashcollateral" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Excess cash collateral</link:label>
    <link:label id="lab_efc_Excesscashcollateral_label_en-US" xlink:label="lab_efc_Excesscashcollateral" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Excess cash collateral</link:label>
    <link:label id="lab_efc_Excesscashcollateral_documentation_en-US" xlink:label="lab_efc_Excesscashcollateral" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Excess cash collateral.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Excesscashcollateral" xlink:href="efc-20201231.xsd#efc_Excesscashcollateral"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Excesscashcollateral" xlink:to="lab_efc_Excesscashcollateral" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedLossBeforeTax_2b434ef6-d135-4e6b-817f-4c2088ca1157_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedLossBeforeTax" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate equity, gross unrealized losses</link:label>
    <link:label id="lab_us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedLossBeforeTax_label_en-US" xlink:label="lab_us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedLossBeforeTax" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Available-for-sale Equity Securities, Accumulated Gross Unrealized Loss, before Tax</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedLossBeforeTax" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedLossBeforeTax"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedLossBeforeTax" xlink:to="lab_us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedLossBeforeTax" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityCentralIndexKey_c38b66fd-2b31-46ec-b73e-27eca07864f1_terseLabel_en-US" xlink:label="lab_dei_EntityCentralIndexKey" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Central Index Key</link:label>
    <link:label id="lab_dei_EntityCentralIndexKey_label_en-US" xlink:label="lab_dei_EntityCentralIndexKey" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Central Index Key</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityCentralIndexKey" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityCentralIndexKey"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityCentralIndexKey" xlink:to="lab_dei_EntityCentralIndexKey" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NetIncomeLossAttributabletoParticipatingNoncontrollingInterest_786f9032-541f-461e-a32d-fee7123665f4_terseLabel_en-US" xlink:label="lab_efc_NetIncomeLossAttributabletoParticipatingNoncontrollingInterest" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Add: Net income (loss) attributable to Convertible Non-controlling Interests</link:label>
    <link:label id="lab_efc_NetIncomeLossAttributabletoParticipatingNoncontrollingInterest_label_en-US" xlink:label="lab_efc_NetIncomeLossAttributabletoParticipatingNoncontrollingInterest" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Income (Loss) Attributable to Participating Non-controlling Interest</link:label>
    <link:label id="lab_efc_NetIncomeLossAttributabletoParticipatingNoncontrollingInterest_documentation_en-US" xlink:label="lab_efc_NetIncomeLossAttributabletoParticipatingNoncontrollingInterest" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net income (loss) attributable to non-controlling interest that owns units convertible to common shares. This income (loss) is included in the calculation of earnings per share. This income (loss) excludes income (loss) attributable to non-controlling interest that does not have the ability to convert their interest into common shares of the Company.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NetIncomeLossAttributabletoParticipatingNoncontrollingInterest" xlink:href="efc-20201231.xsd#efc_NetIncomeLossAttributabletoParticipatingNoncontrollingInterest"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NetIncomeLossAttributabletoParticipatingNoncontrollingInterest" xlink:to="lab_efc_NetIncomeLossAttributabletoParticipatingNoncontrollingInterest" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ValuationAbstract_741d8ac7-8591-42e7-8a54-21aebaded964_terseLabel_en-US" xlink:label="lab_efc_ValuationAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Valuation [Abstract]</link:label>
    <link:label id="lab_efc_ValuationAbstract_label_en-US" xlink:label="lab_efc_ValuationAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Valuation [Abstract]</link:label>
    <link:label id="lab_efc_ValuationAbstract_documentation_en-US" xlink:label="lab_efc_ValuationAbstract" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Valuation [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ValuationAbstract" xlink:href="efc-20201231.xsd#efc_ValuationAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ValuationAbstract" xlink:to="lab_efc_ValuationAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EnergyMember_988316d3-a186-4f7b-aae2-91abb027fb78_terseLabel_en-US" xlink:label="lab_efc_EnergyMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Energy [Member]</link:label>
    <link:label id="lab_efc_EnergyMember_label_en-US" xlink:label="lab_efc_EnergyMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Energy [Member]</link:label>
    <link:label id="lab_efc_EnergyMember_documentation_en-US" xlink:label="lab_efc_EnergyMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Energy [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EnergyMember" xlink:href="efc-20201231.xsd#efc_EnergyMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EnergyMember" xlink:to="lab_efc_EnergyMember" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityEmergingGrowthCompany_025957cf-fd2a-4025-9bf6-bafb9ff2bc6d_terseLabel_en-US" xlink:label="lab_dei_EntityEmergingGrowthCompany" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Emerging Growth Company</link:label>
    <link:label id="lab_dei_EntityEmergingGrowthCompany_label_en-US" xlink:label="lab_dei_EntityEmergingGrowthCompany" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Emerging Growth Company</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityEmergingGrowthCompany" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityEmergingGrowthCompany"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityEmergingGrowthCompany" xlink:to="lab_dei_EntityEmergingGrowthCompany" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_TheCompanyMember_817c6e1c-8972-48c8-b884-bb66c802f088_terseLabel_en-US" xlink:label="lab_efc_TheCompanyMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The Company [Member]</link:label>
    <link:label id="lab_efc_TheCompanyMember_label_en-US" xlink:label="lab_efc_TheCompanyMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The Company [Member]</link:label>
    <link:label id="lab_efc_TheCompanyMember_documentation_en-US" xlink:label="lab_efc_TheCompanyMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The Company (Ellington Financial LLC, the Operating Partnership, and their consolidated subsidiaries) [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_TheCompanyMember" xlink:href="efc-20201231.xsd#efc_TheCompanyMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_TheCompanyMember" xlink:to="lab_efc_TheCompanyMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Numberofmortgageloansinprocessofforeclosure_ac837d1e-2291-4731-8b87-e0f01a58bdc6_terseLabel_en-US" xlink:label="lab_efc_Numberofmortgageloansinprocessofforeclosure" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of mortgage loans in process of foreclosure</link:label>
    <link:label id="lab_efc_Numberofmortgageloansinprocessofforeclosure_label_en-US" xlink:label="lab_efc_Numberofmortgageloansinprocessofforeclosure" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of mortgage loans in process of foreclosure</link:label>
    <link:label id="lab_efc_Numberofmortgageloansinprocessofforeclosure_documentation_en-US" xlink:label="lab_efc_Numberofmortgageloansinprocessofforeclosure" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of mortgage loans in process of foreclosure</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Numberofmortgageloansinprocessofforeclosure" xlink:href="efc-20201231.xsd#efc_Numberofmortgageloansinprocessofforeclosure"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Numberofmortgageloansinprocessofforeclosure" xlink:to="lab_efc_Numberofmortgageloansinprocessofforeclosure" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Maturity31to60daysMember_343a20f8-8eac-4b71-a495-e31e8c1ef379_terseLabel_en-US" xlink:label="lab_efc_Maturity31to60daysMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">31-60 Days</link:label>
    <link:label id="lab_efc_Maturity31to60daysMember_label_en-US" xlink:label="lab_efc_Maturity31to60daysMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity 31 to 60 days [Member]</link:label>
    <link:label id="lab_efc_Maturity31to60daysMember_documentation_en-US" xlink:label="lab_efc_Maturity31to60daysMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity 31 to 60 days [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Maturity31to60daysMember" xlink:href="efc-20201231.xsd#efc_Maturity31to60daysMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Maturity31to60daysMember" xlink:to="lab_efc_Maturity31to60daysMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PurchaseofCertificates_af3245ba-090a-477d-8c6e-cac785abb07e_terseLabel_en-US" xlink:label="lab_efc_PurchaseofCertificates" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchase of certificates</link:label>
    <link:label id="lab_efc_PurchaseofCertificates_label_en-US" xlink:label="lab_efc_PurchaseofCertificates" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchase of Certificates</link:label>
    <link:label id="lab_efc_PurchaseofCertificates_documentation_en-US" xlink:label="lab_efc_PurchaseofCertificates" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchase of Certificates</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PurchaseofCertificates" xlink:href="efc-20201231.xsd#efc_PurchaseofCertificates"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PurchaseofCertificates" xlink:to="lab_efc_PurchaseofCertificates" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PaymentsToMinorityShareholders_1996cdc8-479c-4da3-a1c8-022dc42cc8e0_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_PaymentsToMinorityShareholders" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Distributions to non-controlling interests</link:label>
    <link:label id="lab_us-gaap_PaymentsToMinorityShareholders_label_en-US" xlink:label="lab_us-gaap_PaymentsToMinorityShareholders" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments to Noncontrolling Interests</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PaymentsToMinorityShareholders" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsToMinorityShareholders"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PaymentsToMinorityShareholders" xlink:to="lab_us-gaap_PaymentsToMinorityShareholders" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeTable_16ded4c7-18ea-4c79-9278-4b4ad0664ff6_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative [Table]</link:label>
    <link:label id="lab_us-gaap_DerivativeTable_label_en-US" xlink:label="lab_us-gaap_DerivativeTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeTable" xlink:to="lab_us-gaap_DerivativeTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentSoldNotYetPurchasedBalancePrincipalAmount_34ad8dbf-0ad1-42d8-b532-10c75fc0d5df_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_InvestmentSoldNotYetPurchasedBalancePrincipalAmount" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Current Principal</link:label>
    <link:label id="lab_us-gaap_InvestmentSoldNotYetPurchasedBalancePrincipalAmount_label_en-US" xlink:label="lab_us-gaap_InvestmentSoldNotYetPurchasedBalancePrincipalAmount" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Sold, Not yet Purchased, Balance, Principal Amount</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentSoldNotYetPurchasedBalancePrincipalAmount" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentSoldNotYetPurchasedBalancePrincipalAmount"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentSoldNotYetPurchasedBalancePrincipalAmount" xlink:to="lab_us-gaap_InvestmentSoldNotYetPurchasedBalancePrincipalAmount" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CorporateloanMember_55f11b18-9907-40c8-a840-6eb1f0247386_terseLabel_en-US" xlink:label="lab_efc_CorporateloanMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate loan [Member]</link:label>
    <link:label id="lab_efc_CorporateloanMember_label_en-US" xlink:label="lab_efc_CorporateloanMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate loan [Member]</link:label>
    <link:label id="lab_efc_CorporateloanMember_documentation_en-US" xlink:label="lab_efc_CorporateloanMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate loan [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CorporateloanMember" xlink:href="efc-20201231.xsd#efc_CorporateloanMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CorporateloanMember" xlink:to="lab_efc_CorporateloanMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncomeTaxPolicyTextBlock_d621ae18-a1d6-4f7f-8485-7d1513922102_verboseLabel_en-US" xlink:label="lab_us-gaap_IncomeTaxPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Income Taxes</link:label>
    <link:label id="lab_us-gaap_IncomeTaxPolicyTextBlock_label_en-US" xlink:label="lab_us-gaap_IncomeTaxPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Income Tax, Policy [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncomeTaxPolicyTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncomeTaxPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncomeTaxPolicyTextBlock" xlink:to="lab_us-gaap_IncomeTaxPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PaymentsOfStockIssuanceCosts_aef6c5ce-f6c1-47ca-ac90-a60d31890dd4_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_PaymentsOfStockIssuanceCosts" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Offering costs paid</link:label>
    <link:label id="lab_us-gaap_PaymentsOfStockIssuanceCosts_label_en-US" xlink:label="lab_us-gaap_PaymentsOfStockIssuanceCosts" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments of Stock Issuance Costs</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PaymentsOfStockIssuanceCosts" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsOfStockIssuanceCosts"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PaymentsOfStockIssuanceCosts" xlink:to="lab_us-gaap_PaymentsOfStockIssuanceCosts" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Ownershippercentageoflimitedliabilitycompanyheldasinvestment_77bac22a-4d28-4065-ba4a-4961342db134_terseLabel_en-US" xlink:label="lab_efc_Ownershippercentageoflimitedliabilitycompanyheldasinvestment" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ownership percentage of limited liability company held as investment</link:label>
    <link:label id="lab_efc_Ownershippercentageoflimitedliabilitycompanyheldasinvestment_label_en-US" xlink:label="lab_efc_Ownershippercentageoflimitedliabilitycompanyheldasinvestment" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ownership percentage of limited liability company held as investment</link:label>
    <link:label id="lab_efc_Ownershippercentageoflimitedliabilitycompanyheldasinvestment_documentation_en-US" xlink:label="lab_efc_Ownershippercentageoflimitedliabilitycompanyheldasinvestment" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ownership percentage of limited liability company held as investment</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Ownershippercentageoflimitedliabilitycompanyheldasinvestment" xlink:href="efc-20201231.xsd#efc_Ownershippercentageoflimitedliabilitycompanyheldasinvestment"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Ownershippercentageoflimitedliabilitycompanyheldasinvestment" xlink:to="lab_efc_Ownershippercentageoflimitedliabilitycompanyheldasinvestment" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociation15Year3Point50PercentMember_4bdc6ab7-395d-47b4-bbf7-8e9ae77fcfef_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation15Year3Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (15 Year) 3 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation15Year3Point50PercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation15Year3Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (15 Year) 3 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation15Year3Point50PercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation15Year3Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (15 Year) 3.50%</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociation15Year3Point50PercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociation15Year3Point50PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociation15Year3Point50PercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociation15Year3Point50PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetSales_307ec6ff-b8a5-415d-a154-149a5bbe1a92_negatedLabel_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetSales" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets, Sales</link:label>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetSales_label_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetSales" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement with Unobservable Inputs Reconciliation, Recurring Basis, Asset, Sales</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetSales" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetSales"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetSales" xlink:to="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetSales" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueInputsLevel2Member_01de0277-192b-448b-8d66-7e1701f9f29e_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueInputsLevel2Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Level 2</link:label>
    <link:label id="lab_us-gaap_FairValueInputsLevel2Member_label_en-US" xlink:label="lab_us-gaap_FairValueInputsLevel2Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Inputs, Level 2 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueInputsLevel2Member" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueInputsLevel2Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueInputsLevel2Member" xlink:to="lab_us-gaap_FairValueInputsLevel2Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NetRealizedLoss_194a757d-191b-4246-bd67-df9893093601_terseLabel_en-US" xlink:label="lab_efc_NetRealizedLoss" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized gains (losses) on securities and loans, net</link:label>
    <link:label id="lab_efc_NetRealizedLoss_label_en-US" xlink:label="lab_efc_NetRealizedLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net realized loss</link:label>
    <link:label id="lab_efc_NetRealizedLoss_documentation_en-US" xlink:label="lab_efc_NetRealizedLoss" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net realized loss</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NetRealizedLoss" xlink:href="efc-20201231.xsd#efc_NetRealizedLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NetRealizedLoss" xlink:to="lab_efc_NetRealizedLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancingReceivablesPeriodPastDueDomain_7712f856-0fde-4b6b-9ff3-84eec1548c96_terseLabel_en-US" xlink:label="lab_us-gaap_FinancingReceivablesPeriodPastDueDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Asset, Period Past Due [Domain]</link:label>
    <link:label id="lab_us-gaap_FinancingReceivablesPeriodPastDueDomain_label_en-US" xlink:label="lab_us-gaap_FinancingReceivablesPeriodPastDueDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Asset, Period Past Due [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancingReceivablesPeriodPastDueDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancingReceivablesPeriodPastDueDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancingReceivablesPeriodPastDueDomain" xlink:to="lab_us-gaap_FinancingReceivablesPeriodPastDueDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MinorityInterestOwnershipPercentageByParent_0fc19819-1f27-4b61-86bc-adb88a079a37_terseLabel_en-US" xlink:label="lab_us-gaap_MinorityInterestOwnershipPercentageByParent" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ownership Percentage</link:label>
    <link:label id="lab_us-gaap_MinorityInterestOwnershipPercentageByParent_label_en-US" xlink:label="lab_us-gaap_MinorityInterestOwnershipPercentageByParent" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noncontrolling Interest, Ownership Percentage by Parent</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MinorityInterestOwnershipPercentageByParent" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MinorityInterestOwnershipPercentageByParent"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MinorityInterestOwnershipPercentageByParent" xlink:to="lab_us-gaap_MinorityInterestOwnershipPercentageByParent" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MeasurementInputprojectedcollateralrecoveriesMember_398877dc-a984-43eb-ae8d-6dbd209b54f6_terseLabel_en-US" xlink:label="lab_efc_MeasurementInputprojectedcollateralrecoveriesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, Projected Collateral Recoveries</link:label>
    <link:label id="lab_efc_MeasurementInputprojectedcollateralrecoveriesMember_label_en-US" xlink:label="lab_efc_MeasurementInputprojectedcollateralrecoveriesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, projected collateral recoveries [Member]</link:label>
    <link:label id="lab_efc_MeasurementInputprojectedcollateralrecoveriesMember_documentation_en-US" xlink:label="lab_efc_MeasurementInputprojectedcollateralrecoveriesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Projected recoveries on collateral loans underlying MBS securities, used as an input to measure fair value.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MeasurementInputprojectedcollateralrecoveriesMember" xlink:href="efc-20201231.xsd#efc_MeasurementInputprojectedcollateralrecoveriesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MeasurementInputprojectedcollateralrecoveriesMember" xlink:to="lab_efc_MeasurementInputprojectedcollateralrecoveriesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ConcentrationRiskByTypeAxis_7a3dabba-2633-4b3a-ab76-fe7c0ef1af83_terseLabel_en-US" xlink:label="lab_us-gaap_ConcentrationRiskByTypeAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk Type [Axis]</link:label>
    <link:label id="lab_us-gaap_ConcentrationRiskByTypeAxis_label_en-US" xlink:label="lab_us-gaap_ConcentrationRiskByTypeAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk Type [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ConcentrationRiskByTypeAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ConcentrationRiskByTypeAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ConcentrationRiskByTypeAxis" xlink:to="lab_us-gaap_ConcentrationRiskByTypeAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DividendsCommonStock_6a3ac241-11a0-43ba-8a18-be4d0544dd96_terseLabel_en-US" xlink:label="lab_us-gaap_DividendsCommonStock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends declared</link:label>
    <link:label id="lab_us-gaap_DividendsCommonStock_label_en-US" xlink:label="lab_us-gaap_DividendsCommonStock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends, Common Stock</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DividendsCommonStock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DividendsCommonStock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DividendsCommonStock" xlink:to="lab_us-gaap_DividendsCommonStock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonControllingEquityInterestinLimitedLiabilityCompanyMember_cf4d7c04-8033-44bb-bbf9-e52eb5b3fa12_terseLabel_en-US" xlink:label="lab_efc_NonControllingEquityInterestinLimitedLiabilityCompanyMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment in unconsolidated entities</link:label>
    <link:label id="lab_efc_NonControllingEquityInterestinLimitedLiabilityCompanyMember_label_en-US" xlink:label="lab_efc_NonControllingEquityInterestinLimitedLiabilityCompanyMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Controlling Equity Interest in Limited Liability Company [Member]</link:label>
    <link:label id="lab_efc_NonControllingEquityInterestinLimitedLiabilityCompanyMember_documentation_en-US" xlink:label="lab_efc_NonControllingEquityInterestinLimitedLiabilityCompanyMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Controlling Equity Interest in Limited Liability Company [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonControllingEquityInterestinLimitedLiabilityCompanyMember" xlink:href="efc-20201231.xsd#efc_NonControllingEquityInterestinLimitedLiabilityCompanyMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonControllingEquityInterestinLimitedLiabilityCompanyMember" xlink:to="lab_efc_NonControllingEquityInterestinLimitedLiabilityCompanyMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OperatingPartnershipMember_a108a290-1f98-4c19-af05-ee21e1ebb5a7_terseLabel_en-US" xlink:label="lab_efc_OperatingPartnershipMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership</link:label>
    <link:label id="lab_efc_OperatingPartnershipMember_label_en-US" xlink:label="lab_efc_OperatingPartnershipMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership [Member]</link:label>
    <link:label id="lab_efc_OperatingPartnershipMember_documentation_en-US" xlink:label="lab_efc_OperatingPartnershipMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OperatingPartnershipMember" xlink:href="efc-20201231.xsd#efc_OperatingPartnershipMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OperatingPartnershipMember" xlink:to="lab_efc_OperatingPartnershipMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersOutOfLevel3_6b2feef2-19ba-4c83-97bc-4b37c2986103_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersOutOfLevel3" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets, Transfers out of Level 3</link:label>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersOutOfLevel3_label_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersOutOfLevel3" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement with Unobservable Inputs Reconciliation, Recurring Basis, Asset, Transfers out of Level 3</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersOutOfLevel3" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersOutOfLevel3"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersOutOfLevel3" xlink:to="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersOutOfLevel3" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2026Member_b89cd38b-aee1-41a1-b9b5-9ae9e01e9767_terseLabel_en-US" xlink:label="lab_efc_A2026Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2026 [Member]</link:label>
    <link:label id="lab_efc_A2026Member_label_en-US" xlink:label="lab_efc_A2026Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2026 [Member]</link:label>
    <link:label id="lab_efc_A2026Member_documentation_en-US" xlink:label="lab_efc_A2026Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2026 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2026Member" xlink:href="efc-20201231.xsd#efc_A2026Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2026Member" xlink:to="lab_efc_A2026Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuredborrowingrecoursefacilitycollateralizedbyunsecuredloanportfolioMember_457c2f6b-72a0-487c-9290-5e9975b969e4_terseLabel_en-US" xlink:label="lab_efc_SecuredborrowingrecoursefacilitycollateralizedbyunsecuredloanportfolioMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Secured borrowing recourse facility collateralized by unsecured loan portfolio</link:label>
    <link:label id="lab_efc_SecuredborrowingrecoursefacilitycollateralizedbyunsecuredloanportfolioMember_label_en-US" xlink:label="lab_efc_SecuredborrowingrecoursefacilitycollateralizedbyunsecuredloanportfolioMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Secured borrowing recourse facility collateralized by unsecured loan portfolio [Member]</link:label>
    <link:label id="lab_efc_SecuredborrowingrecoursefacilitycollateralizedbyunsecuredloanportfolioMember_documentation_en-US" xlink:label="lab_efc_SecuredborrowingrecoursefacilitycollateralizedbyunsecuredloanportfolioMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Secured borrowing recourse facility collateralized by unsecured loan portfolio [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuredborrowingrecoursefacilitycollateralizedbyunsecuredloanportfolioMember" xlink:href="efc-20201231.xsd#efc_SecuredborrowingrecoursefacilitycollateralizedbyunsecuredloanportfolioMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuredborrowingrecoursefacilitycollateralizedbyunsecuredloanportfolioMember" xlink:to="lab_efc_SecuredborrowingrecoursefacilitycollateralizedbyunsecuredloanportfolioMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PreferredStockSharesIssued_aa77e9d7-aef4-41e1-bcaa-288584d0c33d_terseLabel_en-US" xlink:label="lab_us-gaap_PreferredStockSharesIssued" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Shares Issued</link:label>
    <link:label id="lab_us-gaap_PreferredStockSharesIssued_label_en-US" xlink:label="lab_us-gaap_PreferredStockSharesIssued" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Shares Issued</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PreferredStockSharesIssued" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PreferredStockSharesIssued"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PreferredStockSharesIssued" xlink:to="lab_us-gaap_PreferredStockSharesIssued" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfRealizedGainLossTableTextBlock_e6176726-45ab-4045-83d1-f9a817f2c2c2_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfRealizedGainLossTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Realized Gain (Loss)</link:label>
    <link:label id="lab_us-gaap_ScheduleOfRealizedGainLossTableTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfRealizedGainLossTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Realized Gain (Loss) [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfRealizedGainLossTableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfRealizedGainLossTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfRealizedGainLossTableTextBlock" xlink:to="lab_us-gaap_ScheduleOfRealizedGainLossTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InvestmentsMaturityDate_e7dec60a-c52a-476d-8eb8-f0735dcef385_terseLabel_en-US" xlink:label="lab_efc_InvestmentsMaturityDate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments maturity date</link:label>
    <link:label id="lab_efc_InvestmentsMaturityDate_label_en-US" xlink:label="lab_efc_InvestmentsMaturityDate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments maturity date</link:label>
    <link:label id="lab_efc_InvestmentsMaturityDate_documentation_en-US" xlink:label="lab_efc_InvestmentsMaturityDate" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments maturity date</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InvestmentsMaturityDate" xlink:href="efc-20201231.xsd#efc_InvestmentsMaturityDate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InvestmentsMaturityDate" xlink:to="lab_efc_InvestmentsMaturityDate" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationFivePercentMember_c158519b-b2f0-41fe-8eb2-b1f9dc7ee32f_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationFivePercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationFivePercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Dayspastdue_0731cfff-d5b4-491b-836f-df0d02bc3bcf_terseLabel_en-US" xlink:label="lab_efc_Dayspastdue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Delinquency (Days)</link:label>
    <link:label id="lab_efc_Dayspastdue_label_en-US" xlink:label="lab_efc_Dayspastdue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Days past due</link:label>
    <link:label id="lab_efc_Dayspastdue_documentation_en-US" xlink:label="lab_efc_Dayspastdue" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of days loan has delinquent payments.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Dayspastdue" xlink:href="efc-20201231.xsd#efc_Dayspastdue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Dayspastdue" xlink:to="lab_efc_Dayspastdue" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MeasurementInputTypeAxis_2bb5266c-8165-463c-8b34-b5b3d227f6b6_terseLabel_en-US" xlink:label="lab_us-gaap_MeasurementInputTypeAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input Type [Axis]</link:label>
    <link:label id="lab_us-gaap_MeasurementInputTypeAxis_label_en-US" xlink:label="lab_us-gaap_MeasurementInputTypeAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input Type [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MeasurementInputTypeAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MeasurementInputTypeAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MeasurementInputTypeAxis" xlink:to="lab_us-gaap_MeasurementInputTypeAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MortgagerelatedCommercialMember_b95603cd-1424-445c-b78e-a23eec479d99_terseLabel_en-US" xlink:label="lab_efc_MortgagerelatedCommercialMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage-related Commercial</link:label>
    <link:label id="lab_efc_MortgagerelatedCommercialMember_label_en-US" xlink:label="lab_efc_MortgagerelatedCommercialMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage-related Commercial [Member]</link:label>
    <link:label id="lab_efc_MortgagerelatedCommercialMember_documentation_en-US" xlink:label="lab_efc_MortgagerelatedCommercialMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage-related Commercial [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MortgagerelatedCommercialMember" xlink:href="efc-20201231.xsd#efc_MortgagerelatedCommercialMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MortgagerelatedCommercialMember" xlink:to="lab_efc_MortgagerelatedCommercialMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OtherinvestmentrelatedexpenseOther_839a8a04-6e2d-49f0-9777-0aeb0cca74ec_terseLabel_en-US" xlink:label="lab_efc_OtherinvestmentrelatedexpenseOther" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other</link:label>
    <link:label id="lab_efc_OtherinvestmentrelatedexpenseOther_label_en-US" xlink:label="lab_efc_OtherinvestmentrelatedexpenseOther" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other investment related expense - Other</link:label>
    <link:label id="lab_efc_OtherinvestmentrelatedexpenseOther_documentation_en-US" xlink:label="lab_efc_OtherinvestmentrelatedexpenseOther" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other investment related expenses not including servicing fees or debt issuance costs related to debt held at fair value.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OtherinvestmentrelatedexpenseOther" xlink:href="efc-20201231.xsd#efc_OtherinvestmentrelatedexpenseOther"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OtherinvestmentrelatedexpenseOther" xlink:to="lab_efc_OtherinvestmentrelatedexpenseOther" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ShareBasedCompensationArrangementByShareBasedPaymentAwardSharesVestedOutstandingNumber_651542f4-5a0f-4481-aaa7-c3e1e9f5966a_terseLabel_en-US" xlink:label="lab_efc_ShareBasedCompensationArrangementByShareBasedPaymentAwardSharesVestedOutstandingNumber" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">OP LTIP Units Vested and Outstanding end of period</link:label>
    <link:label id="lab_efc_ShareBasedCompensationArrangementByShareBasedPaymentAwardSharesVestedOutstandingNumber_label_en-US" xlink:label="lab_efc_ShareBasedCompensationArrangementByShareBasedPaymentAwardSharesVestedOutstandingNumber" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-Based Compensation Arrangement By Share-Based Payment Award, Shares Vested, Outstanding Number</link:label>
    <link:label id="lab_efc_ShareBasedCompensationArrangementByShareBasedPaymentAwardSharesVestedOutstandingNumber_documentation_en-US" xlink:label="lab_efc_ShareBasedCompensationArrangementByShareBasedPaymentAwardSharesVestedOutstandingNumber" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">As of the balance sheet date, the number of shares into which fully vested LTIPs outstanding can be converted into common stock.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ShareBasedCompensationArrangementByShareBasedPaymentAwardSharesVestedOutstandingNumber" xlink:href="efc-20201231.xsd#efc_ShareBasedCompensationArrangementByShareBasedPaymentAwardSharesVestedOutstandingNumber"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ShareBasedCompensationArrangementByShareBasedPaymentAwardSharesVestedOutstandingNumber" xlink:to="lab_efc_ShareBasedCompensationArrangementByShareBasedPaymentAwardSharesVestedOutstandingNumber" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancingReceivables30To59DaysPastDueMember_29e6152a-728f-4a6c-88f3-9b7c3c2a9631_terseLabel_en-US" xlink:label="lab_us-gaap_FinancingReceivables30To59DaysPastDueMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">30 to 59 Days Past Due [Member]</link:label>
    <link:label id="lab_us-gaap_FinancingReceivables30To59DaysPastDueMember_label_en-US" xlink:label="lab_us-gaap_FinancingReceivables30To59DaysPastDueMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Asset, 30 to 59 Days Past Due [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancingReceivables30To59DaysPastDueMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancingReceivables30To59DaysPastDueMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancingReceivables30To59DaysPastDueMember" xlink:to="lab_us-gaap_FinancingReceivables30To59DaysPastDueMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NetCashProvidedByUsedInFinancingActivitiesAbstract_b6891551-a2cb-4bf2-8f3a-a8b9e20a4c57_terseLabel_en-US" xlink:label="lab_us-gaap_NetCashProvidedByUsedInFinancingActivitiesAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash flows provided by (used in) financing activities:</link:label>
    <link:label id="lab_us-gaap_NetCashProvidedByUsedInFinancingActivitiesAbstract_label_en-US" xlink:label="lab_us-gaap_NetCashProvidedByUsedInFinancingActivitiesAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Cash Provided by (Used in) Financing Activities [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NetCashProvidedByUsedInFinancingActivitiesAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NetCashProvidedByUsedInFinancingActivitiesAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NetCashProvidedByUsedInFinancingActivitiesAbstract" xlink:to="lab_us-gaap_NetCashProvidedByUsedInFinancingActivitiesAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_UndistributedEarnings_8c12a1a2-3603-45c2-aec8-40aa786eec83_terseLabel_en-US" xlink:label="lab_us-gaap_UndistributedEarnings" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total undistributed (distributed in excess of) earnings</link:label>
    <link:label id="lab_us-gaap_UndistributedEarnings_label_en-US" xlink:label="lab_us-gaap_UndistributedEarnings" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Undistributed Earnings, Basic</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_UndistributedEarnings" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_UndistributedEarnings"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_UndistributedEarnings" xlink:to="lab_us-gaap_UndistributedEarnings" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection_8ee2a02b-75f3-4793-a0e1-1f009c345cb2_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Amount</link:label>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection_label_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Sold under Agreements to Repurchase, Offset Against Collateral, Net of Not Subject to Master Netting Arrangement, Policy Election</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:to="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeContractTypeDomain_1204cc55-8f26-4568-8661-7939ca3d5476_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeContractTypeDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Contract [Domain]</link:label>
    <link:label id="lab_us-gaap_DerivativeContractTypeDomain_label_en-US" xlink:label="lab_us-gaap_DerivativeContractTypeDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Contract [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeContractTypeDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeContractTypeDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeContractTypeDomain" xlink:to="lab_us-gaap_DerivativeContractTypeDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue_81071784-af03-463c-8c7e-779a17876d20_terseLabel_en-US" xlink:label="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt issuance costs related to Other secured borrowings, at fair value</link:label>
    <link:label id="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue_label_en-US" xlink:label="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt issuance costs related to Other secured borrowings, at fair value</link:label>
    <link:label id="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue_documentation_en-US" xlink:label="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt issuance costs related to Other secured borrowings, at fair value</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue" xlink:href="efc-20201231.xsd#efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue" xlink:to="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsAbstract_5fc1a25a-3cc8-420d-8b1e-8abce404d9f8_terseLabel_en-US" xlink:label="lab_us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Organization, Consolidation and Presentation of Financial Statements [Abstract]</link:label>
    <link:label id="lab_us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsAbstract_label_en-US" xlink:label="lab_us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Organization, Consolidation and Presentation of Financial Statements [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsAbstract" xlink:to="lab_us-gaap_OrganizationConsolidationAndPresentationOfFinancialStatementsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OtherPayablesToBrokerDealersAndClearingOrganizations_17faabed-7905-47d4-b135-2c18cef736fd_terseLabel_en-US" xlink:label="lab_us-gaap_OtherPayablesToBrokerDealersAndClearingOrganizations" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Due to brokers</link:label>
    <link:label id="lab_us-gaap_OtherPayablesToBrokerDealersAndClearingOrganizations_label_en-US" xlink:label="lab_us-gaap_OtherPayablesToBrokerDealersAndClearingOrganizations" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Payables to Broker-Dealers and Clearing Organizations</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OtherPayablesToBrokerDealersAndClearingOrganizations" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OtherPayablesToBrokerDealersAndClearingOrganizations"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OtherPayablesToBrokerDealersAndClearingOrganizations" xlink:to="lab_us-gaap_OtherPayablesToBrokerDealersAndClearingOrganizations" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InterestRateSwapsVariableratesonlyMember_dee5ce1d-c9ec-4ef4-ba2e-6043655127bd_terseLabel_en-US" xlink:label="lab_efc_InterestRateSwapsVariableratesonlyMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Rate Swaps Variable rates only [Member]</link:label>
    <link:label id="lab_efc_InterestRateSwapsVariableratesonlyMember_label_en-US" xlink:label="lab_efc_InterestRateSwapsVariableratesonlyMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Rate Swaps Variable rates only [Member]</link:label>
    <link:label id="lab_efc_InterestRateSwapsVariableratesonlyMember_documentation_en-US" xlink:label="lab_efc_InterestRateSwapsVariableratesonlyMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Rate Swaps Variable rates only [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InterestRateSwapsVariableratesonlyMember" xlink:href="efc-20201231.xsd#efc_InterestRateSwapsVariableratesonlyMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InterestRateSwapsVariableratesonlyMember" xlink:to="lab_efc_InterestRateSwapsVariableratesonlyMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureDisposals_05b15136-07dc-4f1d-94fe-325bdff9060b_negatedTerseLabel_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureDisposals" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disposals</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureDisposals_label_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureDisposals" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure, Disposals</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureDisposals_documentation_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureDisposals" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure, Disposals</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RealEstateAcquiredThroughForeclosureDisposals" xlink:href="efc-20201231.xsd#efc_RealEstateAcquiredThroughForeclosureDisposals"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RealEstateAcquiredThroughForeclosureDisposals" xlink:to="lab_efc_RealEstateAcquiredThroughForeclosureDisposals" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LongTermDebt_4505fadc-8305-40d1-b282-37e125e385a4_terseLabel_en-US" xlink:label="lab_us-gaap_LongTermDebt" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Senior notes, net</link:label>
    <link:label id="lab_us-gaap_LongTermDebt_label_en-US" xlink:label="lab_us-gaap_LongTermDebt" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LongTermDebt" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LongTermDebt"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LongTermDebt" xlink:to="lab_us-gaap_LongTermDebt" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_TransfersAndServicingOfFinancialAssetsTransfersOfFinancialAssetsSalesPolicy_fdb21ffc-cf45-42f2-ac27-682f9093c7f2_terseLabel_en-US" xlink:label="lab_us-gaap_TransfersAndServicingOfFinancialAssetsTransfersOfFinancialAssetsSalesPolicy" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transfers of Financial Assets</link:label>
    <link:label id="lab_us-gaap_TransfersAndServicingOfFinancialAssetsTransfersOfFinancialAssetsSalesPolicy_label_en-US" xlink:label="lab_us-gaap_TransfersAndServicingOfFinancialAssetsTransfersOfFinancialAssetsSalesPolicy" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transfers and Servicing of Financial Assets, Transfers of Financial Assets, Sales, Policy [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_TransfersAndServicingOfFinancialAssetsTransfersOfFinancialAssetsSalesPolicy" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_TransfersAndServicingOfFinancialAssetsTransfersOfFinancialAssetsSalesPolicy"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_TransfersAndServicingOfFinancialAssetsTransfersOfFinancialAssetsSalesPolicy" xlink:to="lab_us-gaap_TransfersAndServicingOfFinancialAssetsTransfersOfFinancialAssetsSalesPolicy" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Otherinvestmentrelatedexpenses_f7af62f8-86c3-4c52-8c2d-04faf9dcf8ee_terseLabel_en-US" xlink:label="lab_efc_Otherinvestmentrelatedexpenses" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other investment related expenses</link:label>
    <link:label id="lab_efc_Otherinvestmentrelatedexpenses_label_en-US" xlink:label="lab_efc_Otherinvestmentrelatedexpenses" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other investment related expenses</link:label>
    <link:label id="lab_efc_Otherinvestmentrelatedexpenses_documentation_en-US" xlink:label="lab_efc_Otherinvestmentrelatedexpenses" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other investment related expenses which can include among other items, dividend expense on short securities and other investment related expenses.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Otherinvestmentrelatedexpenses" xlink:href="efc-20201231.xsd#efc_Otherinvestmentrelatedexpenses"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Otherinvestmentrelatedexpenses" xlink:to="lab_efc_Otherinvestmentrelatedexpenses" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OverAllotmentOptionMember_e1121e4e-2163-4eec-986a-9dad72ece27a_terseLabel_en-US" xlink:label="lab_us-gaap_OverAllotmentOptionMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Over-Allotment Option [Member]</link:label>
    <link:label id="lab_us-gaap_OverAllotmentOptionMember_label_en-US" xlink:label="lab_us-gaap_OverAllotmentOptionMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Over-Allotment Option [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OverAllotmentOptionMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OverAllotmentOptionMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OverAllotmentOptionMember" xlink:to="lab_us-gaap_OverAllotmentOptionMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingAfterYearFive_321dce45-d6b3-4413-9b1b-3b524266b9e9_terseLabel_en-US" xlink:label="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingAfterYearFive" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Maturities, Repayments of Principal in Rolling after Year Five</link:label>
    <link:label id="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingAfterYearFive_label_en-US" xlink:label="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingAfterYearFive" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Maturities, Repayments of Principal in Rolling after Year Five</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingAfterYearFive" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingAfterYearFive"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingAfterYearFive" xlink:to="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingAfterYearFive" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueMeasurementsFairValueHierarchyDomain_76679d8c-334f-4a65-a524-278bd8248eec_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueMeasurementsFairValueHierarchyDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurements, Fair Value Hierarchy [Domain]</link:label>
    <link:label id="lab_us-gaap_FairValueMeasurementsFairValueHierarchyDomain_label_en-US" xlink:label="lab_us-gaap_FairValueMeasurementsFairValueHierarchyDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Hierarchy and NAV [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueMeasurementsFairValueHierarchyDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueMeasurementsFairValueHierarchyDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueMeasurementsFairValueHierarchyDomain" xlink:to="lab_us-gaap_FairValueMeasurementsFairValueHierarchyDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CreditRiskDerivativeinExcessofThresholdatFairValueNet_cb98669c-10db-46c4-bf78-06eb500c18a3_negatedTerseLabel_en-US" xlink:label="lab_efc_CreditRiskDerivativeinExcessofThresholdatFairValueNet" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit risk derivative in excess of threshold, at fair value, net</link:label>
    <link:label id="lab_efc_CreditRiskDerivativeinExcessofThresholdatFairValueNet_label_en-US" xlink:label="lab_efc_CreditRiskDerivativeinExcessofThresholdatFairValueNet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Risk Derivative in Excess of Threshold, at Fair Value, Net</link:label>
    <link:label id="lab_efc_CreditRiskDerivativeinExcessofThresholdatFairValueNet_documentation_en-US" xlink:label="lab_efc_CreditRiskDerivativeinExcessofThresholdatFairValueNet" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net fair value of credit risk derivatives in excess of implied credit spread threshold.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CreditRiskDerivativeinExcessofThresholdatFairValueNet" xlink:href="efc-20201231.xsd#efc_CreditRiskDerivativeinExcessofThresholdatFairValueNet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CreditRiskDerivativeinExcessofThresholdatFairValueNet" xlink:to="lab_efc_CreditRiskDerivativeinExcessofThresholdatFairValueNet" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DepositsAssets_7d30a067-a025-4a1e-9c06-79125ddb7855_terseLabel_en-US" xlink:label="lab_us-gaap_DepositsAssets" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Due from brokers</link:label>
    <link:label id="lab_us-gaap_DepositsAssets_label_en-US" xlink:label="lab_us-gaap_DepositsAssets" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deposits Assets</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DepositsAssets" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DepositsAssets"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DepositsAssets" xlink:to="lab_us-gaap_DepositsAssets" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Impliedspreadonwrittencreditderivative_db4a96ad-7968-44d1-9359-ba1450a101b1_terseLabel_en-US" xlink:label="lab_efc_Impliedspreadonwrittencreditderivative" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Spread on written credit derivatives</link:label>
    <link:label id="lab_efc_Impliedspreadonwrittencreditderivative_label_en-US" xlink:label="lab_efc_Impliedspreadonwrittencreditderivative" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Implied spread on written credit derivative</link:label>
    <link:label id="lab_efc_Impliedspreadonwrittencreditderivative_documentation_en-US" xlink:label="lab_efc_Impliedspreadonwrittencreditderivative" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Implied spread on written credit derivative</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Impliedspreadonwrittencreditderivative" xlink:href="efc-20201231.xsd#efc_Impliedspreadonwrittencreditderivative"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Impliedspreadonwrittencreditderivative" xlink:to="lab_efc_Impliedspreadonwrittencreditderivative" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_VariableInterestEntityNotPrimaryBeneficiaryMember_836da575-de20-4027-88c1-689d51951145_terseLabel_en-US" xlink:label="lab_us-gaap_VariableInterestEntityNotPrimaryBeneficiaryMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Variable Interest Entity, Not Primary Beneficiary</link:label>
    <link:label id="lab_us-gaap_VariableInterestEntityNotPrimaryBeneficiaryMember_label_en-US" xlink:label="lab_us-gaap_VariableInterestEntityNotPrimaryBeneficiaryMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Variable Interest Entity, Not Primary Beneficiary [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_VariableInterestEntityNotPrimaryBeneficiaryMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_VariableInterestEntityNotPrimaryBeneficiaryMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_VariableInterestEntityNotPrimaryBeneficiaryMember" xlink:to="lab_us-gaap_VariableInterestEntityNotPrimaryBeneficiaryMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellGross_d3e6b1eb-da2c-4f1d-867c-05a27c87177c_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellGross" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Purchased under Agreements to Resell, Gross</link:label>
    <link:label id="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellGross_label_en-US" xlink:label="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellGross" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Purchased under Agreements to Resell, Gross</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesPurchasedUnderAgreementsToResellGross" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesPurchasedUnderAgreementsToResellGross"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesPurchasedUnderAgreementsToResellGross" xlink:to="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellGross" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquityMethodInvestmentsDisclosureTextBlock_a9e806ee-48dc-4b4c-aeaa-5ad7cadd81a6_terseLabel_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentsDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments in Unconsolidated Entities</link:label>
    <link:label id="lab_us-gaap_EquityMethodInvestmentsDisclosureTextBlock_label_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentsDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investments and Joint Ventures Disclosure [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquityMethodInvestmentsDisclosureTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquityMethodInvestmentsDisclosureTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquityMethodInvestmentsDisclosureTextBlock" xlink:to="lab_us-gaap_EquityMethodInvestmentsDisclosureTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ConsumerLoanMember_f104b6ad-7416-4f23-9df2-b1087d218368_terseLabel_en-US" xlink:label="lab_us-gaap_ConsumerLoanMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consumer loans</link:label>
    <link:label id="lab_us-gaap_ConsumerLoanMember_label_en-US" xlink:label="lab_us-gaap_ConsumerLoanMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consumer Loan [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ConsumerLoanMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ConsumerLoanMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ConsumerLoanMember" xlink:to="lab_us-gaap_ConsumerLoanMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Concentrationriskpercentagecashandcashequivalents_7ff495b1-231c-4f8c-80dd-f121be4dcf15_terseLabel_en-US" xlink:label="lab_efc_Concentrationriskpercentagecashandcashequivalents" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration risk percentage, cash and cash equivalents</link:label>
    <link:label id="lab_efc_Concentrationriskpercentagecashandcashequivalents_label_en-US" xlink:label="lab_efc_Concentrationriskpercentagecashandcashequivalents" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration risk percentage, cash and cash equivalents</link:label>
    <link:label id="lab_efc_Concentrationriskpercentagecashandcashequivalents_documentation_en-US" xlink:label="lab_efc_Concentrationriskpercentagecashandcashequivalents" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The percentage of cash or cash equivalents held with a specific counterparty.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Concentrationriskpercentagecashandcashequivalents" xlink:href="efc-20201231.xsd#efc_Concentrationriskpercentagecashandcashequivalents"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Concentrationriskpercentagecashandcashequivalents" xlink:to="lab_efc_Concentrationriskpercentagecashandcashequivalents" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationNetincomelossbeforeincometax_ea00f28f-558f-4ca1-a47b-1f77bdc05351_terseLabel_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationNetincomelossbeforeincometax" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Net income (loss) before income tax</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationNetincomelossbeforeincometax_label_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationNetincomelossbeforeincometax" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Net income (loss) before income tax</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationNetincomelossbeforeincometax_documentation_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationNetincomelossbeforeincometax" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The amount of net income (loss) reported by an equity method investment of the entity.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationNetincomelossbeforeincometax" xlink:href="efc-20201231.xsd#efc_EquityMethodInvestmentSummarizedFinancialInformationNetincomelossbeforeincometax"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationNetincomelossbeforeincometax" xlink:to="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationNetincomelossbeforeincometax" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_BlackRockLiquidityFundsFedFundPortfolioInstitutionalClassSharesMember_387a80aa-4efd-4bc2-b619-acac00ded94d_terseLabel_en-US" xlink:label="lab_efc_BlackRockLiquidityFundsFedFundPortfolioInstitutionalClassSharesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">BlackRock Liquidity Funds FedFund Portfolio, Institutional Class Shares [Member]</link:label>
    <link:label id="lab_efc_BlackRockLiquidityFundsFedFundPortfolioInstitutionalClassSharesMember_label_en-US" xlink:label="lab_efc_BlackRockLiquidityFundsFedFundPortfolioInstitutionalClassSharesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">BlackRock Liquidity Funds FedFund Portfolio, Institutional Class Shares [Member]</link:label>
    <link:label id="lab_efc_BlackRockLiquidityFundsFedFundPortfolioInstitutionalClassSharesMember_documentation_en-US" xlink:label="lab_efc_BlackRockLiquidityFundsFedFundPortfolioInstitutionalClassSharesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">BlackRock Liquidity Funds FedFund Portfolio, Institutional Class Shares [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_BlackRockLiquidityFundsFedFundPortfolioInstitutionalClassSharesMember" xlink:href="efc-20201231.xsd#efc_BlackRockLiquidityFundsFedFundPortfolioInstitutionalClassSharesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_BlackRockLiquidityFundsFedFundPortfolioInstitutionalClassSharesMember" xlink:to="lab_efc_BlackRockLiquidityFundsFedFundPortfolioInstitutionalClassSharesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SubsidiarySaleOfStockAxis_5f75afc5-dde2-4f1e-a957-4b99aee69a0f_terseLabel_en-US" xlink:label="lab_us-gaap_SubsidiarySaleOfStockAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Sale of Stock [Axis]</link:label>
    <link:label id="lab_us-gaap_SubsidiarySaleOfStockAxis_label_en-US" xlink:label="lab_us-gaap_SubsidiarySaleOfStockAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Sale of Stock [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SubsidiarySaleOfStockAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SubsidiarySaleOfStockAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SubsidiarySaleOfStockAxis" xlink:to="lab_us-gaap_SubsidiarySaleOfStockAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ValuationTechniqueAxis_f1a8a731-b222-4789-90bc-98aeb6a62d39_terseLabel_en-US" xlink:label="lab_us-gaap_ValuationTechniqueAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Valuation Approach and Technique [Axis]</link:label>
    <link:label id="lab_us-gaap_ValuationTechniqueAxis_label_en-US" xlink:label="lab_us-gaap_ValuationTechniqueAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Valuation Approach and Technique [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ValuationTechniqueAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ValuationTechniqueAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ValuationTechniqueAxis" xlink:to="lab_us-gaap_ValuationTechniqueAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SubsequentEventTable_f9e08512-7473-425c-8861-b06fadd64d07_terseLabel_en-US" xlink:label="lab_us-gaap_SubsequentEventTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsequent Event [Table]</link:label>
    <link:label id="lab_us-gaap_SubsequentEventTable_label_en-US" xlink:label="lab_us-gaap_SubsequentEventTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsequent Event [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SubsequentEventTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SubsequentEventTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SubsequentEventTable" xlink:to="lab_us-gaap_SubsequentEventTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeAssets_b36ed550-7c41-41e4-9a2a-a780f3e888d0_verboseLabel_en-US" xlink:label="lab_us-gaap_DerivativeAssets" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial derivatives&#8212;assets, at fair value</link:label>
    <link:label id="lab_us-gaap_DerivativeAssets_164a2e82-e54c-4669-84b6-abb5e3df1748_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeAssets" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial derivatives&#8211;assets, at fair value-</link:label>
    <link:label id="lab_us-gaap_DerivativeAssets_label_en-US" xlink:label="lab_us-gaap_DerivativeAssets" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Asset</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeAssets" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeAssets"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeAssets" xlink:to="lab_us-gaap_DerivativeAssets" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ToBeAnnouncedSecuritiesLineItems_fee25f0c-b514-4c1e-85c0-5bd259db2368_terseLabel_en-US" xlink:label="lab_efc_ToBeAnnouncedSecuritiesLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">To Be Announced Securities [Line Items]</link:label>
    <link:label id="lab_efc_ToBeAnnouncedSecuritiesLineItems_label_en-US" xlink:label="lab_efc_ToBeAnnouncedSecuritiesLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">To Be Announced Securities [Line Items]</link:label>
    <link:label id="lab_efc_ToBeAnnouncedSecuritiesLineItems_documentation_en-US" xlink:label="lab_efc_ToBeAnnouncedSecuritiesLineItems" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Line Items] for To Be Announced Securities [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ToBeAnnouncedSecuritiesLineItems" xlink:href="efc-20201231.xsd#efc_ToBeAnnouncedSecuritiesLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ToBeAnnouncedSecuritiesLineItems" xlink:to="lab_efc_ToBeAnnouncedSecuritiesLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AccountingPoliciesAbstract_077c1872-e672-4b14-9d7e-af39dccb0f5d_terseLabel_en-US" xlink:label="lab_us-gaap_AccountingPoliciesAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accounting Policies [Abstract]</link:label>
    <link:label id="lab_us-gaap_AccountingPoliciesAbstract_label_en-US" xlink:label="lab_us-gaap_AccountingPoliciesAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accounting Policies [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AccountingPoliciesAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AccountingPoliciesAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AccountingPoliciesAbstract" xlink:to="lab_us-gaap_AccountingPoliciesAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Warehousefacilitymonetaryamount_006aae39-bb13-4e15-9f3a-39c7dbd89cd1_terseLabel_en-US" xlink:label="lab_efc_Warehousefacilitymonetaryamount" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Warehouse facility, monetary amount</link:label>
    <link:label id="lab_efc_Warehousefacilitymonetaryamount_label_en-US" xlink:label="lab_efc_Warehousefacilitymonetaryamount" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Warehouse facility, monetary amount</link:label>
    <link:label id="lab_efc_Warehousefacilitymonetaryamount_documentation_en-US" xlink:label="lab_efc_Warehousefacilitymonetaryamount" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Monetary amount available under warehouse facility</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Warehousefacilitymonetaryamount" xlink:href="efc-20201231.xsd#efc_Warehousefacilitymonetaryamount"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Warehousefacilitymonetaryamount" xlink:to="lab_efc_Warehousefacilitymonetaryamount" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ExpenseReimbursementManager_8b9e0288-8d2a-439c-a22a-6c2065e889a1_terseLabel_en-US" xlink:label="lab_efc_ExpenseReimbursementManager" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Expense Reimbursement - Manager</link:label>
    <link:label id="lab_efc_ExpenseReimbursementManager_label_en-US" xlink:label="lab_efc_ExpenseReimbursementManager" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Expense Reimbursement - Manager</link:label>
    <link:label id="lab_efc_ExpenseReimbursementManager_documentation_en-US" xlink:label="lab_efc_ExpenseReimbursementManager" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Manager reimbursement for operating and compensation expenses related to the Company.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ExpenseReimbursementManager" xlink:href="efc-20201231.xsd#efc_ExpenseReimbursementManager"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ExpenseReimbursementManager" xlink:to="lab_efc_ExpenseReimbursementManager" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMethodInvestmentsSummarizedFinancialInformationInvestments_914a5c58-b88c-4cef-be35-ec713a0e869f_terseLabel_en-US" xlink:label="lab_efc_EquityMethodInvestmentsSummarizedFinancialInformationInvestments" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investments, Summarized Financial Information, Investments</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentsSummarizedFinancialInformationInvestments_label_en-US" xlink:label="lab_efc_EquityMethodInvestmentsSummarizedFinancialInformationInvestments" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investments, Summarized Financial Information, Investments</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentsSummarizedFinancialInformationInvestments_documentation_en-US" xlink:label="lab_efc_EquityMethodInvestmentsSummarizedFinancialInformationInvestments" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The amount of investments reported by an equity method investment of the entity.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMethodInvestmentsSummarizedFinancialInformationInvestments" xlink:href="efc-20201231.xsd#efc_EquityMethodInvestmentsSummarizedFinancialInformationInvestments"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMethodInvestmentsSummarizedFinancialInformationInvestments" xlink:to="lab_efc_EquityMethodInvestmentsSummarizedFinancialInformationInvestments" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentIncomeInterest_44bad60e-942d-45f0-953c-7891ba4644ca_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentIncomeInterest" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Coupon Interest</link:label>
    <link:label id="lab_us-gaap_InvestmentIncomeInterest_label_en-US" xlink:label="lab_us-gaap_InvestmentIncomeInterest" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Income, Interest</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentIncomeInterest" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentIncomeInterest"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentIncomeInterest" xlink:to="lab_us-gaap_InvestmentIncomeInterest" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InvestmentFairValueMeasurementInputPercent_6961fb14-afde-41ec-8d94-4ad857b548d2_terseLabel_en-US" xlink:label="lab_efc_InvestmentFairValueMeasurementInputPercent" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment, measurement input, percent</link:label>
    <link:label id="lab_efc_InvestmentFairValueMeasurementInputPercent_label_en-US" xlink:label="lab_efc_InvestmentFairValueMeasurementInputPercent" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Fair Value Measurement Input, Percent</link:label>
    <link:label id="lab_efc_InvestmentFairValueMeasurementInputPercent_documentation_en-US" xlink:label="lab_efc_InvestmentFairValueMeasurementInputPercent" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment fair value measurement input (investment can include but is not limited to securities, equity, loans, and real estate). Shown as a percent.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InvestmentFairValueMeasurementInputPercent" xlink:href="efc-20201231.xsd#efc_InvestmentFairValueMeasurementInputPercent"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InvestmentFairValueMeasurementInputPercent" xlink:to="lab_efc_InvestmentFairValueMeasurementInputPercent" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluenoncash_49c76acd-0134-4ffc-97a4-8c5c6fe2183a_terseLabel_en-US" xlink:label="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluenoncash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt issuance costs related to Other secured borrowings at fair value non cash</link:label>
    <link:label id="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluenoncash_label_en-US" xlink:label="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluenoncash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt issuance costs related to Other secured borrowings at fair value non cash</link:label>
    <link:label id="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluenoncash_documentation_en-US" xlink:label="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluenoncash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt issuance costs related to Other secured borrowings at fair value non cash</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluenoncash" xlink:href="efc-20201231.xsd#efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluenoncash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluenoncash" xlink:to="lab_efc_DebtissuancecostsrelatedtoOthersecuredborrowingsatfairvaluenoncash" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FixedRateResidentialMortgages20YearMember_afda2448-7587-4db7-86bc-536ebe067c0d_terseLabel_en-US" xlink:label="lab_efc_FixedRateResidentialMortgages20YearMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">20-year fixed-rate mortgages</link:label>
    <link:label id="lab_efc_FixedRateResidentialMortgages20YearMember_label_en-US" xlink:label="lab_efc_FixedRateResidentialMortgages20YearMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed Rate Residential Mortgages, 20 Year [Member]</link:label>
    <link:label id="lab_efc_FixedRateResidentialMortgages20YearMember_documentation_en-US" xlink:label="lab_efc_FixedRateResidentialMortgages20YearMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed Rate Residential Mortgages, 20 Year [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FixedRateResidentialMortgages20YearMember" xlink:href="efc-20201231.xsd#efc_FixedRateResidentialMortgages20YearMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FixedRateResidentialMortgages20YearMember" xlink:to="lab_efc_FixedRateResidentialMortgages20YearMember" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansOnRealEstateNumberOfLoans_58299360-c4cc-4b50-b731-2933921bc1ee_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateNumberOfLoans" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Number of Loans</link:label>
    <link:label id="lab_srt_MortgageLoansOnRealEstateNumberOfLoans_label_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateNumberOfLoans" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Number of Loans</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansOnRealEstateNumberOfLoans" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansOnRealEstateNumberOfLoans"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansOnRealEstateNumberOfLoans" xlink:to="lab_srt_MortgageLoansOnRealEstateNumberOfLoans" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointTwoEightPercentMember_59c521af-5847-4dad-a694-2a63a511e2a8_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointTwoEightPercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Three Point Two Eight Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointTwoEightPercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointTwoEightPercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Three Point Two Eight Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointTwoEightPercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointTwoEightPercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Three Point Two Eight Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointTwoEightPercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointTwoEightPercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointTwoEightPercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePointTwoEightPercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OffsettingOfAssetsAndLiabilitiesAbstract_49ea1889-ace2-4cd7-96f8-98d18310ad05_terseLabel_en-US" xlink:label="lab_efc_OffsettingOfAssetsAndLiabilitiesAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Offsetting of Assets and Liabilities [Abstract]</link:label>
    <link:label id="lab_efc_OffsettingOfAssetsAndLiabilitiesAbstract_label_en-US" xlink:label="lab_efc_OffsettingOfAssetsAndLiabilitiesAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Offsetting of Assets and Liabilities [Abstract]</link:label>
    <link:label id="lab_efc_OffsettingOfAssetsAndLiabilitiesAbstract_documentation_en-US" xlink:label="lab_efc_OffsettingOfAssetsAndLiabilitiesAbstract" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Offsetting of Assets and Liabilities [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OffsettingOfAssetsAndLiabilitiesAbstract" xlink:href="efc-20201231.xsd#efc_OffsettingOfAssetsAndLiabilitiesAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OffsettingOfAssetsAndLiabilitiesAbstract" xlink:to="lab_efc_OffsettingOfAssetsAndLiabilitiesAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CreditDefaultSwapSellingProtectionMember_fdfb8b88-2ee9-4838-aa76-eb946e556a7a_terseLabel_en-US" xlink:label="lab_us-gaap_CreditDefaultSwapSellingProtectionMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swap, Selling Protection [Member]</link:label>
    <link:label id="lab_us-gaap_CreditDefaultSwapSellingProtectionMember_label_en-US" xlink:label="lab_us-gaap_CreditDefaultSwapSellingProtectionMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swap, Selling Protection [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CreditDefaultSwapSellingProtectionMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CreditDefaultSwapSellingProtectionMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CreditDefaultSwapSellingProtectionMember" xlink:to="lab_us-gaap_CreditDefaultSwapSellingProtectionMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MorganStanleyMember_d8a15043-5bec-4ca2-b597-e0734d5eee7a_terseLabel_en-US" xlink:label="lab_efc_MorganStanleyMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Morgan Stanley [Member]</link:label>
    <link:label id="lab_efc_MorganStanleyMember_label_en-US" xlink:label="lab_efc_MorganStanleyMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Morgan Stanley [Member]</link:label>
    <link:label id="lab_efc_MorganStanleyMember_documentation_en-US" xlink:label="lab_efc_MorganStanleyMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Morgan Stanley [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MorganStanleyMember" xlink:href="efc-20201231.xsd#efc_MorganStanleyMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MorganStanleyMember" xlink:to="lab_efc_MorganStanleyMember" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_PayablesToBrokerDealersAndClearingOrganizations_89d88add-3129-4411-b7c2-d6c5c90ce059_verboseLabel_en-US" xlink:label="lab_srt_PayablesToBrokerDealersAndClearingOrganizations" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment related payables</link:label>
    <link:label id="lab_srt_PayablesToBrokerDealersAndClearingOrganizations_4284b608-e418-46b6-a499-f26763908101_negatedTerseLabel_en-US" xlink:label="lab_srt_PayablesToBrokerDealersAndClearingOrganizations" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payable for securities purchased relating to unsettled TBA purchases</link:label>
    <link:label id="lab_srt_PayablesToBrokerDealersAndClearingOrganizations_label_en-US" xlink:label="lab_srt_PayablesToBrokerDealersAndClearingOrganizations" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Broker-Dealer, Payable to Other Broker-Dealer and Clearing Organization</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_PayablesToBrokerDealersAndClearingOrganizations" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_PayablesToBrokerDealersAndClearingOrganizations"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_PayablesToBrokerDealersAndClearingOrganizations" xlink:to="lab_srt_PayablesToBrokerDealersAndClearingOrganizations" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Writtencreditderivativespreadthreshold_76a3c537-6fc3-4a36-bb8a-233be83077d0_terseLabel_en-US" xlink:label="lab_efc_Writtencreditderivativespreadthreshold" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Written credit derivative spread threshold</link:label>
    <link:label id="lab_efc_Writtencreditderivativespreadthreshold_label_en-US" xlink:label="lab_efc_Writtencreditderivativespreadthreshold" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Written credit derivative spread threshold</link:label>
    <link:label id="lab_efc_Writtencreditderivativespreadthreshold_documentation_en-US" xlink:label="lab_efc_Writtencreditderivativespreadthreshold" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Implied credit spread threshold whereby the contracts trade based on points paid up front.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Writtencreditderivativespreadthreshold" xlink:href="efc-20201231.xsd#efc_Writtencreditderivativespreadthreshold"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Writtencreditderivativespreadthreshold" xlink:to="lab_efc_Writtencreditderivativespreadthreshold" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentsFairValueDisclosure_74ddcc26-b4a9-40c7-8077-e8d4385443ba_verboseLabel_en-US" xlink:label="lab_us-gaap_InvestmentsFairValueDisclosure" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments, at fair value-</link:label>
    <link:label id="lab_us-gaap_InvestmentsFairValueDisclosure_label_en-US" xlink:label="lab_us-gaap_InvestmentsFairValueDisclosure" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments, Fair Value Disclosure</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentsFairValueDisclosure" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentsFairValueDisclosure"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentsFairValueDisclosure" xlink:to="lab_us-gaap_InvestmentsFairValueDisclosure" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LoansReceivableMember_66365990-fb57-4998-a93c-9f379071f20a_terseLabel_en-US" xlink:label="lab_us-gaap_LoansReceivableMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans</link:label>
    <link:label id="lab_us-gaap_LoansReceivableMember_label_en-US" xlink:label="lab_us-gaap_LoansReceivableMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans Receivable [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LoansReceivableMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LoansReceivableMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LoansReceivableMember" xlink:to="lab_us-gaap_LoansReceivableMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_IncentiveFeeLossCarryforward_4fef463a-982f-4401-8e7a-034180cf74db_terseLabel_en-US" xlink:label="lab_efc_IncentiveFeeLossCarryforward" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Incentive Fee, Loss Carryforward</link:label>
    <link:label id="lab_efc_IncentiveFeeLossCarryforward_label_en-US" xlink:label="lab_efc_IncentiveFeeLossCarryforward" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Incentive Fee, Loss Carryforward</link:label>
    <link:label id="lab_efc_IncentiveFeeLossCarryforward_documentation_en-US" xlink:label="lab_efc_IncentiveFeeLossCarryforward" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Incentive Fee, Loss Carryforward</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_IncentiveFeeLossCarryforward" xlink:href="efc-20201231.xsd#efc_IncentiveFeeLossCarryforward"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_IncentiveFeeLossCarryforward" xlink:to="lab_efc_IncentiveFeeLossCarryforward" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityAddressAddressLine1_38c5d940-cf8a-4d11-9ed2-2c06078860bc_terseLabel_en-US" xlink:label="lab_dei_EntityAddressAddressLine1" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Address, Address Line One</link:label>
    <link:label id="lab_dei_EntityAddressAddressLine1_label_en-US" xlink:label="lab_dei_EntityAddressAddressLine1" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Address, Address Line One</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityAddressAddressLine1" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityAddressAddressLine1"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityAddressAddressLine1" xlink:to="lab_dei_EntityAddressAddressLine1" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ToBeAnnouncedFixedRateAgencySecuritiesMember_6b5cb210-6f7c-4988-a6be-1dc1c5b0dbed_terseLabel_en-US" xlink:label="lab_efc_ToBeAnnouncedFixedRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">To Be Announced Fixed Rate Agency Securities [Member]</link:label>
    <link:label id="lab_efc_ToBeAnnouncedFixedRateAgencySecuritiesMember_label_en-US" xlink:label="lab_efc_ToBeAnnouncedFixedRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">To Be Announced Fixed Rate Agency Securities [Member]</link:label>
    <link:label id="lab_efc_ToBeAnnouncedFixedRateAgencySecuritiesMember_documentation_en-US" xlink:label="lab_efc_ToBeAnnouncedFixedRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">A forward settling security issued by a government agency with a fixed interest rate.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ToBeAnnouncedFixedRateAgencySecuritiesMember" xlink:href="efc-20201231.xsd#efc_ToBeAnnouncedFixedRateAgencySecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ToBeAnnouncedFixedRateAgencySecuritiesMember" xlink:to="lab_efc_ToBeAnnouncedFixedRateAgencySecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritiesTransactionsAndInvestmentIncomePolicyPolicyTextBlock_22ee3c37-354f-4692-91c8-71e52aad2c54_terseLabel_en-US" xlink:label="lab_efc_SecuritiesTransactionsAndInvestmentIncomePolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Transactions And Investment Income Policy [Policy Text Block]</link:label>
    <link:label id="lab_efc_SecuritiesTransactionsAndInvestmentIncomePolicyPolicyTextBlock_label_en-US" xlink:label="lab_efc_SecuritiesTransactionsAndInvestmentIncomePolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Transactions And Investment Income Policy [Policy Text Block]</link:label>
    <link:label id="lab_efc_SecuritiesTransactionsAndInvestmentIncomePolicyPolicyTextBlock_documentation_en-US" xlink:label="lab_efc_SecuritiesTransactionsAndInvestmentIncomePolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disclosure of accounting policy for recording securities transactions, calculation of realized and unrealized gains, and policy regarding accretion of discounts and amortization of premiums.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritiesTransactionsAndInvestmentIncomePolicyPolicyTextBlock" xlink:href="efc-20201231.xsd#efc_SecuritiesTransactionsAndInvestmentIncomePolicyPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritiesTransactionsAndInvestmentIncomePolicyPolicyTextBlock" xlink:to="lab_efc_SecuritiesTransactionsAndInvestmentIncomePolicyPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_SegmentGeographicalDomain_2d826cfc-fafb-4925-a501-17d9056b3cf0_terseLabel_en-US" xlink:label="lab_srt_SegmentGeographicalDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Geographical [Domain]</link:label>
    <link:label id="lab_srt_SegmentGeographicalDomain_label_en-US" xlink:label="lab_srt_SegmentGeographicalDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Geographical [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_SegmentGeographicalDomain" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_SegmentGeographicalDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_SegmentGeographicalDomain" xlink:to="lab_srt_SegmentGeographicalDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DerivativeNotional_79abc0c7-fa54-43b2-9843-29f43d676bdc_terseLabel_en-US" xlink:label="lab_efc_DerivativeNotional" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative notional</link:label>
    <link:label id="lab_efc_DerivativeNotional_43a1e0a2-e2b1-498e-9732-0e7903f19adf_negatedTerseLabel_en-US" xlink:label="lab_efc_DerivativeNotional" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative notional</link:label>
    <link:label id="lab_efc_DerivativeNotional_label_en-US" xlink:label="lab_efc_DerivativeNotional" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative notional</link:label>
    <link:label id="lab_efc_DerivativeNotional_documentation_en-US" xlink:label="lab_efc_DerivativeNotional" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative notional amount</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DerivativeNotional" xlink:href="efc-20201231.xsd#efc_DerivativeNotional"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DerivativeNotional" xlink:to="lab_efc_DerivativeNotional" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LoansReceivableFairValueDisclosure_2d908b40-0d19-492e-a8be-ddb72d161477_verboseLabel_en-US" xlink:label="lab_us-gaap_LoansReceivableFairValueDisclosure" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans, at fair value</link:label>
    <link:label id="lab_us-gaap_LoansReceivableFairValueDisclosure_label_en-US" xlink:label="lab_us-gaap_LoansReceivableFairValueDisclosure" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans Receivable, Fair Value Disclosure</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LoansReceivableFairValueDisclosure" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LoansReceivableFairValueDisclosure"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LoansReceivableFairValueDisclosure" xlink:to="lab_us-gaap_LoansReceivableFairValueDisclosure" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202311MemberDomain_d4e4ecb2-dc02-45a2-8475-aedf36963707_terseLabel_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202311MemberDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.88 Percent, Maturity Date 2023 11 [Member] [Domain]</link:label>
    <link:label id="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202311MemberDomain_label_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202311MemberDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.88 Percent, Maturity Date 2023 11 [Member] [Domain]</link:label>
    <link:label id="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202311MemberDomain_documentation_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202311MemberDomain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.88 Percent, Maturity Date 2023 11 [Member] [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_U.S.TreasuryNote2.88PercentMaturityDate202311MemberDomain" xlink:href="efc-20201231.xsd#efc_U.S.TreasuryNote2.88PercentMaturityDate202311MemberDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_U.S.TreasuryNote2.88PercentMaturityDate202311MemberDomain" xlink:to="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202311MemberDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityAddressStateOrProvince_f4b6c05f-4af7-4a61-9a0e-dea96b50aa53_terseLabel_en-US" xlink:label="lab_dei_EntityAddressStateOrProvince" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Address, State or Province</link:label>
    <link:label id="lab_dei_EntityAddressStateOrProvince_label_en-US" xlink:label="lab_dei_EntityAddressStateOrProvince" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Address, State or Province</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityAddressStateOrProvince" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityAddressStateOrProvince"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityAddressStateOrProvince" xlink:to="lab_dei_EntityAddressStateOrProvince" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonaccrualstatusMember_42b902df-03b9-471f-bf73-617cbcffcd89_terseLabel_en-US" xlink:label="lab_efc_NonaccrualstatusMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-accrual status [Member]</link:label>
    <link:label id="lab_efc_NonaccrualstatusMember_label_en-US" xlink:label="lab_efc_NonaccrualstatusMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-accrual status [Member]</link:label>
    <link:label id="lab_efc_NonaccrualstatusMember_documentation_en-US" xlink:label="lab_efc_NonaccrualstatusMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans not accruing interest</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonaccrualstatusMember" xlink:href="efc-20201231.xsd#efc_NonaccrualstatusMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonaccrualstatusMember" xlink:to="lab_efc_NonaccrualstatusMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationRevenue_63aa984a-596c-43e0-ae7a-eb97c82840bf_terseLabel_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationRevenue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Revenue</link:label>
    <link:label id="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationRevenue_label_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationRevenue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Revenue</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationRevenue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquityMethodInvestmentSummarizedFinancialInformationRevenue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationRevenue" xlink:to="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationRevenue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OthersecuredborrowingsMember_2ba80b40-1a1d-4ae1-8d34-2dd56b5f304a_terseLabel_en-US" xlink:label="lab_efc_OthersecuredborrowingsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other secured borrowings, at fair value</link:label>
    <link:label id="lab_efc_OthersecuredborrowingsMember_b1f2f0f1-0fb8-4bff-b477-5434bb678ae1_verboseLabel_en-US" xlink:label="lab_efc_OthersecuredborrowingsMember" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Secured Borrowings</link:label>
    <link:label id="lab_efc_OthersecuredborrowingsMember_label_en-US" xlink:label="lab_efc_OthersecuredborrowingsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other secured borrowings [Member]</link:label>
    <link:label id="lab_efc_OthersecuredborrowingsMember_documentation_en-US" xlink:label="lab_efc_OthersecuredborrowingsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other secured borrowings include secured financing accounted for as collateralized borrowings.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OthersecuredborrowingsMember" xlink:href="efc-20201231.xsd#efc_OthersecuredborrowingsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OthersecuredborrowingsMember" xlink:to="lab_efc_OthersecuredborrowingsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NetRealizedGainLossOnDerivatives_04ba56b9-2c3f-4c0d-b8ba-9e0b5e50ba26_terseLabel_en-US" xlink:label="lab_efc_NetRealizedGainLossOnDerivatives" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized (gains) losses on financial derivatives, net</link:label>
    <link:label id="lab_efc_NetRealizedGainLossOnDerivatives_0179b9d5-2b45-4ccd-ace4-778347382321_negatedTerseLabel_en-US" xlink:label="lab_efc_NetRealizedGainLossOnDerivatives" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized (gains) losses on financial derivatives, net</link:label>
    <link:label id="lab_efc_NetRealizedGainLossOnDerivatives_label_en-US" xlink:label="lab_efc_NetRealizedGainLossOnDerivatives" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Realized Gain/Loss on Derivatives</link:label>
    <link:label id="lab_efc_NetRealizedGainLossOnDerivatives_documentation_en-US" xlink:label="lab_efc_NetRealizedGainLossOnDerivatives" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Realized Gain/Loss on Derivatives</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NetRealizedGainLossOnDerivatives" xlink:href="efc-20201231.xsd#efc_NetRealizedGainLossOnDerivatives"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NetRealizedGainLossOnDerivatives" xlink:to="lab_efc_NetRealizedGainLossOnDerivatives" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss_0353ef88-d994-45fa-abda-a232366ed4b5_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized Losses, Less than 12 Months</link:label>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss_label_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Continuous Unrealized Loss Position, Less than 12 Months, Accumulated Loss</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss" xlink:to="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12MonthsAccumulatedLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_WeightedAverageMember_474dcac7-eac9-4a2b-affc-4e8d7651f83c_terseLabel_en-US" xlink:label="lab_srt_WeightedAverageMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Weighted Average</link:label>
    <link:label id="lab_srt_WeightedAverageMember_label_en-US" xlink:label="lab_srt_WeightedAverageMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Weighted Average [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_WeightedAverageMember" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_WeightedAverageMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_WeightedAverageMember" xlink:to="lab_srt_WeightedAverageMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NetCashProvidedByUsedInInvestingActivities_b8e93ea6-993a-4f4a-b358-5999f4559e00_totalLabel_en-US" xlink:label="lab_us-gaap_NetCashProvidedByUsedInInvestingActivities" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net cash provided by (used in) investing activities</link:label>
    <link:label id="lab_us-gaap_NetCashProvidedByUsedInInvestingActivities_label_en-US" xlink:label="lab_us-gaap_NetCashProvidedByUsedInInvestingActivities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Cash Provided by (Used in) Investing Activities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NetCashProvidedByUsedInInvestingActivities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NetCashProvidedByUsedInInvestingActivities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NetCashProvidedByUsedInInvestingActivities" xlink:to="lab_us-gaap_NetCashProvidedByUsedInInvestingActivities" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InterestOnlyStripMember_60352999-1464-4fd3-9da2-b6445531b184_terseLabel_en-US" xlink:label="lab_us-gaap_InterestOnlyStripMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest only securities</link:label>
    <link:label id="lab_us-gaap_InterestOnlyStripMember_label_en-US" xlink:label="lab_us-gaap_InterestOnlyStripMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest-Only-Strip [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InterestOnlyStripMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InterestOnlyStripMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InterestOnlyStripMember" xlink:to="lab_us-gaap_InterestOnlyStripMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisTable_3e7f2ec0-53f4-470a-9d3a-629c3145785b_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Recurring and Nonrecurring [Table]</link:label>
    <link:label id="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisTable_label_en-US" xlink:label="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Recurring and Nonrecurring [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisTable" xlink:to="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RelatedPartyTransactionLineItems_f1b4e204-5647-4693-9eb7-98220f47f333_terseLabel_en-US" xlink:label="lab_us-gaap_RelatedPartyTransactionLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party Transaction [Line Items]</link:label>
    <link:label id="lab_us-gaap_RelatedPartyTransactionLineItems_label_en-US" xlink:label="lab_us-gaap_RelatedPartyTransactionLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party Transaction [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RelatedPartyTransactionLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RelatedPartyTransactionLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RelatedPartyTransactionLineItems" xlink:to="lab_us-gaap_RelatedPartyTransactionLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ConsolidatedStatementofOperationsParentheticalAbstract_7e042a09-5eed-41c6-a328-0bab9d956a16_terseLabel_en-US" xlink:label="lab_efc_ConsolidatedStatementofOperationsParentheticalAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidated Statement of Operations (Parenthetical) [Abstract]</link:label>
    <link:label id="lab_efc_ConsolidatedStatementofOperationsParentheticalAbstract_label_en-US" xlink:label="lab_efc_ConsolidatedStatementofOperationsParentheticalAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidated Statement of Operations (Parenthetical) [Abstract]</link:label>
    <link:label id="lab_efc_ConsolidatedStatementofOperationsParentheticalAbstract_documentation_en-US" xlink:label="lab_efc_ConsolidatedStatementofOperationsParentheticalAbstract" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidated Statement of Operations (Parenthetical) [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ConsolidatedStatementofOperationsParentheticalAbstract" xlink:href="efc-20201231.xsd#efc_ConsolidatedStatementofOperationsParentheticalAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ConsolidatedStatementofOperationsParentheticalAbstract" xlink:to="lab_efc_ConsolidatedStatementofOperationsParentheticalAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ResetCLOISecuritizationMember_1e82b102-18d9-4189-862c-1e478ecd5091_terseLabel_en-US" xlink:label="lab_efc_ResetCLOISecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reset CLO I Securitization [Member]</link:label>
    <link:label id="lab_efc_ResetCLOISecuritizationMember_label_en-US" xlink:label="lab_efc_ResetCLOISecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reset CLO I Securitization [Member]</link:label>
    <link:label id="lab_efc_ResetCLOISecuritizationMember_documentation_en-US" xlink:label="lab_efc_ResetCLOISecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reset CLO I Securitization (reset in 08 2018) [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ResetCLOISecuritizationMember" xlink:href="efc-20201231.xsd#efc_ResetCLOISecuritizationMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ResetCLOISecuritizationMember" xlink:to="lab_efc_ResetCLOISecuritizationMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_November152017Member_ab9e75f8-5ea0-4989-896a-dbe2a61ac78d_terseLabel_en-US" xlink:label="lab_efc_November152017Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">November 2017 [Member]</link:label>
    <link:label id="lab_efc_November152017Member_label_en-US" xlink:label="lab_efc_November152017Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">November 15, 2017 [Member]</link:label>
    <link:label id="lab_efc_November152017Member_documentation_en-US" xlink:label="lab_efc_November152017Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">November 15, 2017 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_November152017Member" xlink:href="efc-20201231.xsd#efc_November152017Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_November152017Member" xlink:to="lab_efc_November152017Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfInvestmentsAbstract_label_en-US" xlink:label="lab_us-gaap_ScheduleOfInvestmentsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Investments [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfInvestmentsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfInvestmentsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfInvestmentsAbstract" xlink:to="lab_us-gaap_ScheduleOfInvestmentsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ForeignCurrencyTranslationGainLossUnrealized_e0c3d618-633b-4a4f-9d64-bf34f18cded4_terseLabel_en-US" xlink:label="lab_efc_ForeignCurrencyTranslationGainLossUnrealized" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized (gains) losses other, net&#8212;foreign currency translation</link:label>
    <link:label id="lab_efc_ForeignCurrencyTranslationGainLossUnrealized_label_en-US" xlink:label="lab_efc_ForeignCurrencyTranslationGainLossUnrealized" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Foreign Currency Translation Gain Loss Unrealized</link:label>
    <link:label id="lab_efc_ForeignCurrencyTranslationGainLossUnrealized_documentation_en-US" xlink:label="lab_efc_ForeignCurrencyTranslationGainLossUnrealized" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount before tax of foreign currency translation unrealized gain (loss) recognized in the income statement.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ForeignCurrencyTranslationGainLossUnrealized" xlink:href="efc-20201231.xsd#efc_ForeignCurrencyTranslationGainLossUnrealized"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ForeignCurrencyTranslationGainLossUnrealized" xlink:to="lab_efc_ForeignCurrencyTranslationGainLossUnrealized" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansOnRealEstateScheduleTable_105f5345-9414-4e7e-9700-c15633413c32_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateScheduleTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate Schedule [Table]</link:label>
    <link:label id="lab_srt_MortgageLoansOnRealEstateScheduleTable_label_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateScheduleTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate Schedule [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansOnRealEstateScheduleTable" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansOnRealEstateScheduleTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansOnRealEstateScheduleTable" xlink:to="lab_srt_MortgageLoansOnRealEstateScheduleTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfDerivativeInstrumentsTextBlock_325fef9c-9e7f-4f68-864a-f7c5e2b736bf_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfDerivativeInstrumentsTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Derivative Instruments</link:label>
    <link:label id="lab_us-gaap_ScheduleOfDerivativeInstrumentsTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfDerivativeInstrumentsTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Derivative Instruments [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfDerivativeInstrumentsTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfDerivativeInstrumentsTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfDerivativeInstrumentsTextBlock" xlink:to="lab_us-gaap_ScheduleOfDerivativeInstrumentsTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SignificantAccountingPoliciesTextBlock_5644b408-2772-4565-bd75-581982a5720f_terseLabel_en-US" xlink:label="lab_us-gaap_SignificantAccountingPoliciesTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Significant Accounting Policies</link:label>
    <link:label id="lab_us-gaap_SignificantAccountingPoliciesTextBlock_label_en-US" xlink:label="lab_us-gaap_SignificantAccountingPoliciesTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Significant Accounting Policies [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SignificantAccountingPoliciesTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SignificantAccountingPoliciesTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SignificantAccountingPoliciesTextBlock" xlink:to="lab_us-gaap_SignificantAccountingPoliciesTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsOtherFivePercentMember_bf2ddb9c-5cb9-4194-bbe6-7a296f72aeee_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsOtherFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Other) Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsOtherFivePercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsOtherFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Other) Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsOtherFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsOtherFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Other) Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsOtherFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsOtherFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsOtherFivePercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsOtherFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember_3c8d0696-ca59-4165-9e09-1f828ca1968e_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Four Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_BankofNewYorkMellonCorporationMember_aef8d3e8-5f31-4532-90e2-cda5816c1ff2_terseLabel_en-US" xlink:label="lab_efc_BankofNewYorkMellonCorporationMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of New York Mellon Corporation [Member]</link:label>
    <link:label id="lab_efc_BankofNewYorkMellonCorporationMember_label_en-US" xlink:label="lab_efc_BankofNewYorkMellonCorporationMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of New York Mellon Corporation [Member]</link:label>
    <link:label id="lab_efc_BankofNewYorkMellonCorporationMember_documentation_en-US" xlink:label="lab_efc_BankofNewYorkMellonCorporationMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of New York Mellon Corporation [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_BankofNewYorkMellonCorporationMember" xlink:href="efc-20201231.xsd#efc_BankofNewYorkMellonCorporationMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_BankofNewYorkMellonCorporationMember" xlink:to="lab_efc_BankofNewYorkMellonCorporationMember" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityInteractiveDataCurrent_843f639d-72df-437e-9222-d694f3818d5b_terseLabel_en-US" xlink:label="lab_dei_EntityInteractiveDataCurrent" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Interactive Data Current</link:label>
    <link:label id="lab_dei_EntityInteractiveDataCurrent_label_en-US" xlink:label="lab_dei_EntityInteractiveDataCurrent" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Interactive Data Current</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityInteractiveDataCurrent" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityInteractiveDataCurrent"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityInteractiveDataCurrent" xlink:to="lab_dei_EntityInteractiveDataCurrent" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_GeographicConcentrationRiskMember_e48c45b8-d092-4d56-b697-8558def607b8_terseLabel_en-US" xlink:label="lab_us-gaap_GeographicConcentrationRiskMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Geographic Concentration Risk [Member]</link:label>
    <link:label id="lab_us-gaap_GeographicConcentrationRiskMember_label_en-US" xlink:label="lab_us-gaap_GeographicConcentrationRiskMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Geographic Concentration Risk [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_GeographicConcentrationRiskMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_GeographicConcentrationRiskMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_GeographicConcentrationRiskMember" xlink:to="lab_us-gaap_GeographicConcentrationRiskMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CommonStockSharesIssued_9d4ccdf7-9caa-4dcd-9bd7-a29ca3d9e012_terseLabel_en-US" xlink:label="lab_us-gaap_CommonStockSharesIssued" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common stock, shares issued</link:label>
    <link:label id="lab_us-gaap_CommonStockSharesIssued_label_en-US" xlink:label="lab_us-gaap_CommonStockSharesIssued" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common Stock, Shares, Issued</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CommonStockSharesIssued" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CommonStockSharesIssued"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CommonStockSharesIssued" xlink:to="lab_us-gaap_CommonStockSharesIssued" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CommonStockParOrStatedValuePerShare_baf69610-9416-477b-b272-f9ddf1c47627_terseLabel_en-US" xlink:label="lab_us-gaap_CommonStockParOrStatedValuePerShare" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common shares</link:label>
    <link:label id="lab_us-gaap_CommonStockParOrStatedValuePerShare_label_en-US" xlink:label="lab_us-gaap_CommonStockParOrStatedValuePerShare" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common Stock, Par or Stated Value Per Share</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CommonStockParOrStatedValuePerShare" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CommonStockParOrStatedValuePerShare"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CommonStockParOrStatedValuePerShare" xlink:to="lab_us-gaap_CommonStockParOrStatedValuePerShare" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MeasurementInputQuotedPriceMember_d89c77d1-21de-4200-ac56-490ab03dc04e_terseLabel_en-US" xlink:label="lab_us-gaap_MeasurementInputQuotedPriceMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, Quoted Price</link:label>
    <link:label id="lab_us-gaap_MeasurementInputQuotedPriceMember_label_en-US" xlink:label="lab_us-gaap_MeasurementInputQuotedPriceMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, Quoted Price [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MeasurementInputQuotedPriceMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MeasurementInputQuotedPriceMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MeasurementInputQuotedPriceMember" xlink:to="lab_us-gaap_MeasurementInputQuotedPriceMember" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityIncorporationStateCountryCode_28401b24-6af9-48e8-ba27-77b0b3dfb6ad_terseLabel_en-US" xlink:label="lab_dei_EntityIncorporationStateCountryCode" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Incorporation, State or Country Code</link:label>
    <link:label id="lab_dei_EntityIncorporationStateCountryCode_label_en-US" xlink:label="lab_dei_EntityIncorporationStateCountryCode" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Incorporation, State or Country Code</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityIncorporationStateCountryCode" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityIncorporationStateCountryCode"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityIncorporationStateCountryCode" xlink:to="lab_dei_EntityIncorporationStateCountryCode" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DividendsAbstract_5af800ff-ce23-4af7-ad72-a9fc44298de4_terseLabel_en-US" xlink:label="lab_us-gaap_DividendsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends Paid:</link:label>
    <link:label id="lab_us-gaap_DividendsAbstract_label_en-US" xlink:label="lab_us-gaap_DividendsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DividendsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DividendsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DividendsAbstract" xlink:to="lab_us-gaap_DividendsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeLiabilityMeasurementInput_94d74782-267a-49ea-bd70-6ad027a53d9d_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeLiabilityMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Liability, Measurement Input</link:label>
    <link:label id="lab_us-gaap_DerivativeLiabilityMeasurementInput_label_en-US" xlink:label="lab_us-gaap_DerivativeLiabilityMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Liability, Measurement Input</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeLiabilityMeasurementInput" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeLiabilityMeasurementInput"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeLiabilityMeasurementInput" xlink:to="lab_us-gaap_DerivativeLiabilityMeasurementInput" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansBetween250000And499999Member_870ac217-a921-41b4-a690-521246e11877_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansBetween250000And499999Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $250,000 and $499,999</link:label>
    <link:label id="lab_srt_MortgageLoansBetween250000And499999Member_label_en-US" xlink:label="lab_srt_MortgageLoansBetween250000And499999Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $250,000 and $499,999 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansBetween250000And499999Member" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansBetween250000And499999Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansBetween250000And499999Member" xlink:to="lab_srt_MortgageLoansBetween250000And499999Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Affiliatedcoparticipantsinitialpercentageownershipofjointlyownedentity_77c3f00b-d217-4d29-9aba-765581cd61c1_terseLabel_en-US" xlink:label="lab_efc_Affiliatedcoparticipantsinitialpercentageownershipofjointlyownedentity" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Affiliated co-participants initial percentage ownership of jointly owned entity</link:label>
    <link:label id="lab_efc_Affiliatedcoparticipantsinitialpercentageownershipofjointlyownedentity_label_en-US" xlink:label="lab_efc_Affiliatedcoparticipantsinitialpercentageownershipofjointlyownedentity" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Affiliated co-participants initial percentage ownership of jointly owned entity</link:label>
    <link:label id="lab_efc_Affiliatedcoparticipantsinitialpercentageownershipofjointlyownedentity_documentation_en-US" xlink:label="lab_efc_Affiliatedcoparticipantsinitialpercentageownershipofjointlyownedentity" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Affiliated co-participant's initial percentage ownership of jointly owned entity with the Company.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Affiliatedcoparticipantsinitialpercentageownershipofjointlyownedentity" xlink:href="efc-20201231.xsd#efc_Affiliatedcoparticipantsinitialpercentageownershipofjointlyownedentity"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Affiliatedcoparticipantsinitialpercentageownershipofjointlyownedentity" xlink:to="lab_efc_Affiliatedcoparticipantsinitialpercentageownershipofjointlyownedentity" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1250000And1499999Member_b6891a0d-ab8f-41e1-8db4-b7a1d39202f3_terseLabel_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1250000And1499999Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $1,250,000 and $1,499,999</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1250000And1499999Member_label_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1250000And1499999Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $1,250,000 and $1,499,999 [Member]</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1250000And1499999Member_documentation_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1250000And1499999Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $1,250,000 and $1,499,999</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1250000And1499999Member" xlink:href="efc-20201231.xsd#efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1250000And1499999Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1250000And1499999Member" xlink:to="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1250000And1499999Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeAssetsAbstract_8d5c8a78-4d2d-4d83-874d-3598e421f0ae_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeAssetsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial derivatives&#8211;assets</link:label>
    <link:label id="lab_us-gaap_DerivativeAssetsAbstract_label_en-US" xlink:label="lab_us-gaap_DerivativeAssetsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Asset [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeAssetsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeAssetsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeAssetsAbstract" xlink:to="lab_us-gaap_DerivativeAssetsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePointFivePercentMember_6a52df6e-4093-405b-80dd-0b05652bba36_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Fifteen Year) Three Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Fifteen Year) Three Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Fifteen Year) Three Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePointFivePercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearThreePointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Grossunrealizedlosses_1b992b4e-39fb-4472-849e-9092f1ded641_negatedTerseLabel_en-US" xlink:label="lab_efc_Grossunrealizedlosses" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Losses</link:label>
    <link:label id="lab_efc_Grossunrealizedlosses_label_en-US" xlink:label="lab_efc_Grossunrealizedlosses" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Gross unrealized losses</link:label>
    <link:label id="lab_efc_Grossunrealizedlosses_documentation_en-US" xlink:label="lab_efc_Grossunrealizedlosses" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Gross unrealized loss on investment.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Grossunrealizedlosses" xlink:href="efc-20201231.xsd#efc_Grossunrealizedlosses"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Grossunrealizedlosses" xlink:to="lab_efc_Grossunrealizedlosses" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeLiabilitiesAbstract_e4f84011-88e1-42b5-9593-460e1551cd3e_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeLiabilitiesAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial derivatives&#8211;liabilities</link:label>
    <link:label id="lab_us-gaap_DerivativeLiabilitiesAbstract_label_en-US" xlink:label="lab_us-gaap_DerivativeLiabilitiesAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Liability [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeLiabilitiesAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeLiabilitiesAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeLiabilitiesAbstract" xlink:to="lab_us-gaap_DerivativeLiabilitiesAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FinancingFacilityTypeDomain_779af4ce-80ef-4f7d-879f-609ba739915a_terseLabel_en-US" xlink:label="lab_efc_FinancingFacilityTypeDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Facility Type [Domain]</link:label>
    <link:label id="lab_efc_FinancingFacilityTypeDomain_label_en-US" xlink:label="lab_efc_FinancingFacilityTypeDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Facility Type [Domain]</link:label>
    <link:label id="lab_efc_FinancingFacilityTypeDomain_documentation_en-US" xlink:label="lab_efc_FinancingFacilityTypeDomain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Domain] for Financing Facility Type [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FinancingFacilityTypeDomain" xlink:href="efc-20201231.xsd#efc_FinancingFacilityTypeDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FinancingFacilityTypeDomain" xlink:to="lab_efc_FinancingFacilityTypeDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InvestmentOwnedAtFairValueNet_5fed4ecc-c451-4827-8ab0-263b8f2b181f_terseLabel_en-US" xlink:label="lab_efc_InvestmentOwnedAtFairValueNet" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Owned At Fair Value, Net</link:label>
    <link:label id="lab_efc_InvestmentOwnedAtFairValueNet_184483d8-093c-42ab-9edf-bfee9ff0f5f1_negatedLabel_en-US" xlink:label="lab_efc_InvestmentOwnedAtFairValueNet" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net short TBA securities, at fair value</link:label>
    <link:label id="lab_efc_InvestmentOwnedAtFairValueNet_label_en-US" xlink:label="lab_efc_InvestmentOwnedAtFairValueNet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Owned At Fair Value, Net</link:label>
    <link:label id="lab_efc_InvestmentOwnedAtFairValueNet_documentation_en-US" xlink:label="lab_efc_InvestmentOwnedAtFairValueNet" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The net fair value of an entity's securities position as of the reporting date.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InvestmentOwnedAtFairValueNet" xlink:href="efc-20201231.xsd#efc_InvestmentOwnedAtFairValueNet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InvestmentOwnedAtFairValueNet" xlink:to="lab_efc_InvestmentOwnedAtFairValueNet" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EurodollarFutureMember_bab64aa9-cf66-411d-b12b-8484cc58e89f_terseLabel_en-US" xlink:label="lab_us-gaap_EurodollarFutureMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Eurodollar futures</link:label>
    <link:label id="lab_us-gaap_EurodollarFutureMember_label_en-US" xlink:label="lab_us-gaap_EurodollarFutureMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Eurodollar Future [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EurodollarFutureMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EurodollarFutureMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EurodollarFutureMember" xlink:to="lab_us-gaap_EurodollarFutureMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancingReceivables60To89DaysPastDueMember_db6ce5ce-23d9-470f-a07e-d5404ebea3e4_terseLabel_en-US" xlink:label="lab_us-gaap_FinancingReceivables60To89DaysPastDueMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">60 to 89 Days Past Due [Member]</link:label>
    <link:label id="lab_us-gaap_FinancingReceivables60To89DaysPastDueMember_label_en-US" xlink:label="lab_us-gaap_FinancingReceivables60To89DaysPastDueMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Asset, 60 to 89 Days Past Due [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancingReceivables60To89DaysPastDueMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancingReceivables60To89DaysPastDueMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancingReceivables60To89DaysPastDueMember" xlink:to="lab_us-gaap_FinancingReceivables60To89DaysPastDueMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncreaseDecreaseInOtherOperatingAssets_a6068832-3539-495a-9add-b641a3014452_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInOtherOperatingAssets" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other assets</link:label>
    <link:label id="lab_us-gaap_IncreaseDecreaseInOtherOperatingAssets_label_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInOtherOperatingAssets" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (Decrease) in Other Operating Assets</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncreaseDecreaseInOtherOperatingAssets" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncreaseDecreaseInOtherOperatingAssets"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncreaseDecreaseInOtherOperatingAssets" xlink:to="lab_us-gaap_IncreaseDecreaseInOtherOperatingAssets" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CommitmentsAndContingenciesDisclosureTextBlock_b5e793e7-e2c0-4de3-9b2b-baaa89797ca2_terseLabel_en-US" xlink:label="lab_us-gaap_CommitmentsAndContingenciesDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Contingencies and Commitments</link:label>
    <link:label id="lab_us-gaap_CommitmentsAndContingenciesDisclosureTextBlock_label_en-US" xlink:label="lab_us-gaap_CommitmentsAndContingenciesDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commitments and Contingencies Disclosure [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CommitmentsAndContingenciesDisclosureTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CommitmentsAndContingenciesDisclosureTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CommitmentsAndContingenciesDisclosureTextBlock" xlink:to="lab_us-gaap_CommitmentsAndContingenciesDisclosureTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeFinancialInstrumentsLiabilitiesMember_49e18a92-83df-41d2-b86b-7f63ea8242e2_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeFinancialInstrumentsLiabilitiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Derivatives - Liabilities</link:label>
    <link:label id="lab_us-gaap_DerivativeFinancialInstrumentsLiabilitiesMember_label_en-US" xlink:label="lab_us-gaap_DerivativeFinancialInstrumentsLiabilitiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Financial Instruments, Liabilities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeFinancialInstrumentsLiabilitiesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeFinancialInstrumentsLiabilitiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeFinancialInstrumentsLiabilitiesMember" xlink:to="lab_us-gaap_DerivativeFinancialInstrumentsLiabilitiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentsSoldNotYetPurchasedTable_de1a3654-ca84-4265-83bd-7c02f56c32eb_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentsSoldNotYetPurchasedTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments Sold, Not yet Purchased [Table]</link:label>
    <link:label id="lab_us-gaap_InvestmentsSoldNotYetPurchasedTable_label_en-US" xlink:label="lab_us-gaap_InvestmentsSoldNotYetPurchasedTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments Sold, Not yet Purchased [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentsSoldNotYetPurchasedTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentsSoldNotYetPurchasedTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentsSoldNotYetPurchasedTable" xlink:to="lab_us-gaap_InvestmentsSoldNotYetPurchasedTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OPTN20201HoldingsLLCMember_95407952-ed1a-4b26-b9b6-596fd9dc7606_terseLabel_en-US" xlink:label="lab_efc_OPTN20201HoldingsLLCMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">OPTN 2020-1 Holdings LLC</link:label>
    <link:label id="lab_efc_OPTN20201HoldingsLLCMember_label_en-US" xlink:label="lab_efc_OPTN20201HoldingsLLCMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">OPTN 2020-1 Holdings LLC [Member]</link:label>
    <link:label id="lab_efc_OPTN20201HoldingsLLCMember_documentation_en-US" xlink:label="lab_efc_OPTN20201HoldingsLLCMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">OPTN 2020-1 Holdings LLC</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OPTN20201HoldingsLLCMember" xlink:href="efc-20201231.xsd#efc_OPTN20201HoldingsLLCMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OPTN20201HoldingsLLCMember" xlink:to="lab_efc_OPTN20201HoldingsLLCMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StockIssuedDuringPeriodSharesNewIssues_17bb4f0d-a112-45c1-927c-01d38444fb20_terseLabel_en-US" xlink:label="lab_us-gaap_StockIssuedDuringPeriodSharesNewIssues" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stock Issued During Period, Shares, New Issues</link:label>
    <link:label id="lab_us-gaap_StockIssuedDuringPeriodSharesNewIssues_label_en-US" xlink:label="lab_us-gaap_StockIssuedDuringPeriodSharesNewIssues" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stock Issued During Period, Shares, New Issues</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StockIssuedDuringPeriodSharesNewIssues" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockIssuedDuringPeriodSharesNewIssues"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StockIssuedDuringPeriodSharesNewIssues" xlink:to="lab_us-gaap_StockIssuedDuringPeriodSharesNewIssues" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ProceedsfromtheDispositionofInvestmentsNonCash_a6012779-a7ab-496e-a1f9-d7d4a284677f_terseLabel_en-US" xlink:label="lab_efc_ProceedsfromtheDispositionofInvestmentsNonCash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from the Disposition of Investments Non Cash</link:label>
    <link:label id="lab_efc_ProceedsfromtheDispositionofInvestmentsNonCash_label_en-US" xlink:label="lab_efc_ProceedsfromtheDispositionofInvestmentsNonCash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from the Disposition of Investments Non Cash</link:label>
    <link:label id="lab_efc_ProceedsfromtheDispositionofInvestmentsNonCash_documentation_en-US" xlink:label="lab_efc_ProceedsfromtheDispositionofInvestmentsNonCash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from the Disposition of Investments Non Cash</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ProceedsfromtheDispositionofInvestmentsNonCash" xlink:href="efc-20201231.xsd#efc_ProceedsfromtheDispositionofInvestmentsNonCash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ProceedsfromtheDispositionofInvestmentsNonCash" xlink:to="lab_efc_ProceedsfromtheDispositionofInvestmentsNonCash" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsMember_464f567b-cc5f-4543-a2be-bf4a0e3e4565_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentCompanyInvestmentIncomeLossRatio_f59bc80c-b755-4dee-8d2a-5671bdab6c31_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentCompanyInvestmentIncomeLossRatio" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Investment Income (Loss) Ratio</link:label>
    <link:label id="lab_us-gaap_InvestmentCompanyInvestmentIncomeLossRatio_label_en-US" xlink:label="lab_us-gaap_InvestmentCompanyInvestmentIncomeLossRatio" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Investment Income (Loss) Ratio</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentCompanyInvestmentIncomeLossRatio" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentCompanyInvestmentIncomeLossRatio"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentCompanyInvestmentIncomeLossRatio" xlink:to="lab_us-gaap_InvestmentCompanyInvestmentIncomeLossRatio" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year4Point50PercentMember_0bfce62e-f019-4dde-875a-fd9e53ecebce_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year4Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 4 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year4Point50PercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year4Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 4 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year4Point50PercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year4Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 4 Point 50 Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociation30Year4Point50PercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociation30Year4Point50PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociation30Year4Point50PercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociation30Year4Point50PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Maturitygreaterthan360daysMember_2d333b78-0791-4e58-94b1-2d0412b161a7_terseLabel_en-US" xlink:label="lab_efc_Maturitygreaterthan360daysMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Credit Assets</link:label>
    <link:label id="lab_efc_Maturitygreaterthan360daysMember_label_en-US" xlink:label="lab_efc_Maturitygreaterthan360daysMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity greater than 360 days [Member]</link:label>
    <link:label id="lab_efc_Maturitygreaterthan360daysMember_documentation_en-US" xlink:label="lab_efc_Maturitygreaterthan360daysMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity greater than 360 days [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Maturitygreaterthan360daysMember" xlink:href="efc-20201231.xsd#efc_Maturitygreaterthan360daysMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Maturitygreaterthan360daysMember" xlink:to="lab_efc_Maturitygreaterthan360daysMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_U.S.TreasuryNote2.75PercentMaturityDate202308Member_b5304bc4-06bc-438d-bd45-f3fa25cde799_terseLabel_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.75PercentMaturityDate202308Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.75 Percent, Maturity Date 2023 08 [Member]</link:label>
    <link:label id="lab_efc_U.S.TreasuryNote2.75PercentMaturityDate202308Member_label_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.75PercentMaturityDate202308Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.75 Percent, Maturity Date 2023 08 [Member]</link:label>
    <link:label id="lab_efc_U.S.TreasuryNote2.75PercentMaturityDate202308Member_documentation_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.75PercentMaturityDate202308Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.75 Percent, Maturity Date 2023 08 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_U.S.TreasuryNote2.75PercentMaturityDate202308Member" xlink:href="efc-20201231.xsd#efc_U.S.TreasuryNote2.75PercentMaturityDate202308Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_U.S.TreasuryNote2.75PercentMaturityDate202308Member" xlink:to="lab_efc_U.S.TreasuryNote2.75PercentMaturityDate202308Member" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_ConsolidatedEntitiesAxis_686ffce6-5031-4e97-8fea-7e86c16a5554_terseLabel_en-US" xlink:label="lab_srt_ConsolidatedEntitiesAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidated Entities [Axis]</link:label>
    <link:label id="lab_srt_ConsolidatedEntitiesAxis_label_en-US" xlink:label="lab_srt_ConsolidatedEntitiesAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidated Entities [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_ConsolidatedEntitiesAxis" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_ConsolidatedEntitiesAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_ConsolidatedEntitiesAxis" xlink:to="lab_srt_ConsolidatedEntitiesAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RatioOfOperatingExpensesBeforeIncentiveFeeInterestExpenseandOtherInvestmentRelatedExpensesToAverageEquity_cd2423a6-77a2-4da2-9a64-03c84a249d7b_negatedTerseLabel_en-US" xlink:label="lab_efc_RatioOfOperatingExpensesBeforeIncentiveFeeInterestExpenseandOtherInvestmentRelatedExpensesToAverageEquity" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ratio Of Operating Expenses Before Incentive Fee, Interest Expense, and Other Investment Related Expenses To Average Equity</link:label>
    <link:label id="lab_efc_RatioOfOperatingExpensesBeforeIncentiveFeeInterestExpenseandOtherInvestmentRelatedExpensesToAverageEquity_label_en-US" xlink:label="lab_efc_RatioOfOperatingExpensesBeforeIncentiveFeeInterestExpenseandOtherInvestmentRelatedExpensesToAverageEquity" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ratio Of Operating Expenses Before Incentive Fee, Interest Expense, and Other Investment Related Expenses To Average Equity</link:label>
    <link:label id="lab_efc_RatioOfOperatingExpensesBeforeIncentiveFeeInterestExpenseandOtherInvestmentRelatedExpensesToAverageEquity_documentation_en-US" xlink:label="lab_efc_RatioOfOperatingExpensesBeforeIncentiveFeeInterestExpenseandOtherInvestmentRelatedExpensesToAverageEquity" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The ratio of operating expenses (before incentive fee, interest expense, and other investment related expenses) to average total equity (inclusive of non-controlling interest).</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RatioOfOperatingExpensesBeforeIncentiveFeeInterestExpenseandOtherInvestmentRelatedExpensesToAverageEquity" xlink:href="efc-20201231.xsd#efc_RatioOfOperatingExpensesBeforeIncentiveFeeInterestExpenseandOtherInvestmentRelatedExpensesToAverageEquity"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RatioOfOperatingExpensesBeforeIncentiveFeeInterestExpenseandOtherInvestmentRelatedExpensesToAverageEquity" xlink:to="lab_efc_RatioOfOperatingExpensesBeforeIncentiveFeeInterestExpenseandOtherInvestmentRelatedExpensesToAverageEquity" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DeferredFederalStateAndLocalTaxExpenseBenefit_e0c5d202-820b-4f15-81ac-2c6b6087d670_terseLabel_en-US" xlink:label="lab_us-gaap_DeferredFederalStateAndLocalTaxExpenseBenefit" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Federal, State and Local, Tax Expense (Benefit)</link:label>
    <link:label id="lab_us-gaap_DeferredFederalStateAndLocalTaxExpenseBenefit_label_en-US" xlink:label="lab_us-gaap_DeferredFederalStateAndLocalTaxExpenseBenefit" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Federal, State and Local, Tax Expense (Benefit)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DeferredFederalStateAndLocalTaxExpenseBenefit" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DeferredFederalStateAndLocalTaxExpenseBenefit"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DeferredFederalStateAndLocalTaxExpenseBenefit" xlink:to="lab_us-gaap_DeferredFederalStateAndLocalTaxExpenseBenefit" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisLineItems_f26189aa-3681-4b48-bee1-87f6dac6d23d_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Assets and Liabilities Measured on Recurring and Nonrecurring Basis [Line Items]</link:label>
    <link:label id="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisLineItems_label_en-US" xlink:label="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Assets and Liabilities Measured on Recurring and Nonrecurring Basis [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisLineItems" xlink:to="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Netincomelossattributabletononcontrollinginterestpershare_8c4dbe3e-0851-43b5-96e9-664757f1a99e_terseLabel_en-US" xlink:label="lab_efc_Netincomelossattributabletononcontrollinginterestpershare" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Less: Results of Operations Attributable to Non-controlling Interests</link:label>
    <link:label id="lab_efc_Netincomelossattributabletononcontrollinginterestpershare_label_en-US" xlink:label="lab_efc_Netincomelossattributabletononcontrollinginterestpershare" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">NetIncomeLossAttributableToNoncontrollingInterestPerShare</link:label>
    <link:label id="lab_efc_Netincomelossattributabletononcontrollinginterestpershare_documentation_en-US" xlink:label="lab_efc_Netincomelossattributabletononcontrollinginterestpershare" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount of Net Income (Loss) attributable to noncontrolling interest based on average common shares outstanding during the period.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Netincomelossattributabletononcontrollinginterestpershare" xlink:href="efc-20201231.xsd#efc_Netincomelossattributabletononcontrollinginterestpershare"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Netincomelossattributabletononcontrollinginterestpershare" xlink:to="lab_efc_Netincomelossattributabletononcontrollinginterestpershare" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MeasurementInputOfferedPriceMember_8db0a018-ba21-4ad4-a7af-d2b54c0b53f1_terseLabel_en-US" xlink:label="lab_us-gaap_MeasurementInputOfferedPriceMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, Offered Price</link:label>
    <link:label id="lab_us-gaap_MeasurementInputOfferedPriceMember_label_en-US" xlink:label="lab_us-gaap_MeasurementInputOfferedPriceMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, Offered Price [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MeasurementInputOfferedPriceMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MeasurementInputOfferedPriceMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MeasurementInputOfferedPriceMember" xlink:to="lab_us-gaap_MeasurementInputOfferedPriceMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPercentMember_ce1d195f-60e3-41b1-ac23-262b879eda3a_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Four Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Four Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Four Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFourPercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleofAssetsandLiabilitiesofConsolidatedSecuritizationTrustsTableTextBlock_e33b66d4-f2d3-4782-92ab-0e469433eb0f_terseLabel_en-US" xlink:label="lab_efc_ScheduleofAssetsandLiabilitiesofConsolidatedSecuritizationTrustsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Assets and Liabilities of Consolidated Securitization Trusts</link:label>
    <link:label id="lab_efc_ScheduleofAssetsandLiabilitiesofConsolidatedSecuritizationTrustsTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleofAssetsandLiabilitiesofConsolidatedSecuritizationTrustsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Assets and Liabilities of Consolidated Securitization Trusts [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleofAssetsandLiabilitiesofConsolidatedSecuritizationTrustsTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleofAssetsandLiabilitiesofConsolidatedSecuritizationTrustsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Tabular disclosure of the amounts included in the Consolidated Statement of Assets, Liabilities, and Equity attributable to securitization trusts that have been consolidated by the Company.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleofAssetsandLiabilitiesofConsolidatedSecuritizationTrustsTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleofAssetsandLiabilitiesofConsolidatedSecuritizationTrustsTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleofAssetsandLiabilitiesofConsolidatedSecuritizationTrustsTableTextBlock" xlink:to="lab_efc_ScheduleofAssetsandLiabilitiesofConsolidatedSecuritizationTrustsTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfEffectiveIncomeTaxRateReconciliationTableTextBlock_47e2189d-e3e6-4364-92bb-f76d769b127e_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfEffectiveIncomeTaxRateReconciliationTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Effective Income Tax Rate Reconciliation</link:label>
    <link:label id="lab_us-gaap_ScheduleOfEffectiveIncomeTaxRateReconciliationTableTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfEffectiveIncomeTaxRateReconciliationTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Effective Income Tax Rate Reconciliation [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfEffectiveIncomeTaxRateReconciliationTableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfEffectiveIncomeTaxRateReconciliationTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfEffectiveIncomeTaxRateReconciliationTableTextBlock" xlink:to="lab_us-gaap_ScheduleOfEffectiveIncomeTaxRateReconciliationTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CommonStockValue_d2dc7d7b-a0df-491d-8bb6-9a8c66a67d22_terseLabel_en-US" xlink:label="lab_us-gaap_CommonStockValue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common stock, par value $0.001 per share, 100,000,000 shares authorized; 43,781,684 and 38,647,943 shares issued and outstanding, respectively</link:label>
    <link:label id="lab_us-gaap_CommonStockValue_label_en-US" xlink:label="lab_us-gaap_CommonStockValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common Stock, Value, Issued</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CommonStockValue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CommonStockValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CommonStockValue" xlink:to="lab_us-gaap_CommonStockValue" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RealEstateOwnedDisclosureOfDetailedComponentsAbstract_194f4424-40cd-474f-b77d-589fbb7574a4_terseLabel_en-US" xlink:label="lab_us-gaap_RealEstateOwnedDisclosureOfDetailedComponentsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Owned, Disclosure of Detailed Components [Abstract]</link:label>
    <link:label id="lab_us-gaap_RealEstateOwnedDisclosureOfDetailedComponentsAbstract_label_en-US" xlink:label="lab_us-gaap_RealEstateOwnedDisclosureOfDetailedComponentsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Owned, Disclosure of Detailed Components [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RealEstateOwnedDisclosureOfDetailedComponentsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RealEstateOwnedDisclosureOfDetailedComponentsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RealEstateOwnedDisclosureOfDetailedComponentsAbstract" xlink:to="lab_us-gaap_RealEstateOwnedDisclosureOfDetailedComponentsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OtherPrivateLabelSecuritiesMember_55705ef8-784f-49f8-a0b7-1b00e1984a6a_terseLabel_en-US" xlink:label="lab_efc_OtherPrivateLabelSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Private Label Securities [Member]</link:label>
    <link:label id="lab_efc_OtherPrivateLabelSecuritiesMember_label_en-US" xlink:label="lab_efc_OtherPrivateLabelSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Private Label Securities [Member]</link:label>
    <link:label id="lab_efc_OtherPrivateLabelSecuritiesMember_documentation_en-US" xlink:label="lab_efc_OtherPrivateLabelSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Private Label Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OtherPrivateLabelSecuritiesMember" xlink:href="efc-20201231.xsd#efc_OtherPrivateLabelSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OtherPrivateLabelSecuritiesMember" xlink:to="lab_efc_OtherPrivateLabelSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonExchangeTradedEquityInvestmentinMortgageOriginatorsMember_d2f8006f-f4e8-44e2-a3e9-526c76e0112d_terseLabel_en-US" xlink:label="lab_efc_NonExchangeTradedEquityInvestmentinMortgageOriginatorsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Equity Investment in Mortgage Originators [Member]</link:label>
    <link:label id="lab_efc_NonExchangeTradedEquityInvestmentinMortgageOriginatorsMember_label_en-US" xlink:label="lab_efc_NonExchangeTradedEquityInvestmentinMortgageOriginatorsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Equity Investment in Mortgage Originators [Member]</link:label>
    <link:label id="lab_efc_NonExchangeTradedEquityInvestmentinMortgageOriginatorsMember_documentation_en-US" xlink:label="lab_efc_NonExchangeTradedEquityInvestmentinMortgageOriginatorsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Equity Investment in Mortgage Originators [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonExchangeTradedEquityInvestmentinMortgageOriginatorsMember" xlink:href="efc-20201231.xsd#efc_NonExchangeTradedEquityInvestmentinMortgageOriginatorsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonExchangeTradedEquityInvestmentinMortgageOriginatorsMember" xlink:to="lab_efc_NonExchangeTradedEquityInvestmentinMortgageOriginatorsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ReceivableForSecuritiesSoldAndFailingTrades_4d8a9d10-ed74-4841-b09d-be029ca5d90c_terseLabel_en-US" xlink:label="lab_efc_ReceivableForSecuritiesSoldAndFailingTrades" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Receivable for securities sold relating to unsettled TBA sales</link:label>
    <link:label id="lab_efc_ReceivableForSecuritiesSoldAndFailingTrades_label_en-US" xlink:label="lab_efc_ReceivableForSecuritiesSoldAndFailingTrades" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Receivable for securities sold and failing trades</link:label>
    <link:label id="lab_efc_ReceivableForSecuritiesSoldAndFailingTrades_documentation_en-US" xlink:label="lab_efc_ReceivableForSecuritiesSoldAndFailingTrades" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Includes receivables due from securities sold prior to period end where the transactions have not yet settled as well as receivables due from securities failed to deliver.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ReceivableForSecuritiesSoldAndFailingTrades" xlink:href="efc-20201231.xsd#efc_ReceivableForSecuritiesSoldAndFailingTrades"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ReceivableForSecuritiesSoldAndFailingTrades" xlink:to="lab_efc_ReceivableForSecuritiesSoldAndFailingTrades" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_VestingDomain_03fdf1fe-d7ed-4b3a-8e86-8acb8166ece4_terseLabel_en-US" xlink:label="lab_us-gaap_VestingDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vesting [Domain]</link:label>
    <link:label id="lab_us-gaap_VestingDomain_label_en-US" xlink:label="lab_us-gaap_VestingDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vesting [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_VestingDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_VestingDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_VestingDomain" xlink:to="lab_us-gaap_VestingDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Fixedrateonderivativeunderlyingoptionscontract_90f94610-8268-4b40-b85b-c63733e8ac82_terseLabel_en-US" xlink:label="lab_efc_Fixedrateonderivativeunderlyingoptionscontract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed rate on derivative underlying options contract</link:label>
    <link:label id="lab_efc_Fixedrateonderivativeunderlyingoptionscontract_label_en-US" xlink:label="lab_efc_Fixedrateonderivativeunderlyingoptionscontract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed rate on derivative underlying options contract</link:label>
    <link:label id="lab_efc_Fixedrateonderivativeunderlyingoptionscontract_documentation_en-US" xlink:label="lab_efc_Fixedrateonderivativeunderlyingoptionscontract" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed rate on derivative underlying options contract</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Fixedrateonderivativeunderlyingoptionscontract" xlink:href="efc-20201231.xsd#efc_Fixedrateonderivativeunderlyingoptionscontract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Fixedrateonderivativeunderlyingoptionscontract" xlink:to="lab_efc_Fixedrateonderivativeunderlyingoptionscontract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationFourPointFivePercentMember_f9af8285-9e8d-43ae-adba-00f5c81bb38f_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationFourPointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationFourPointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Four Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationFourPointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationFourPointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationFourPointFivePercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationFourPointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Netincomelossattributabletoparentandparticipatingnoncontrollinginterest_11be62ef-03bf-44b2-962e-027c451d985d_totalLabel_en-US" xlink:label="lab_efc_Netincomelossattributabletoparentandparticipatingnoncontrollinginterest" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net income (loss) related to common stockholders and Convertible Non-controlling Interests</link:label>
    <link:label id="lab_efc_Netincomelossattributabletoparentandparticipatingnoncontrollinginterest_label_en-US" xlink:label="lab_efc_Netincomelossattributabletoparentandparticipatingnoncontrollinginterest" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net income (loss) attributable to parent and participating non-controlling interest</link:label>
    <link:label id="lab_efc_Netincomelossattributabletoparentandparticipatingnoncontrollinginterest_documentation_en-US" xlink:label="lab_efc_Netincomelossattributabletoparentandparticipatingnoncontrollinginterest" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net income (loss) attributable to parent and participating non-controlling interest. Income (loss) excludes income (loss) attributable to non-controlling interest that does not have the ability to convert their interest into common shares of the Company.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Netincomelossattributabletoparentandparticipatingnoncontrollinginterest" xlink:href="efc-20201231.xsd#efc_Netincomelossattributabletoparentandparticipatingnoncontrollinginterest"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Netincomelossattributabletoparentandparticipatingnoncontrollinginterest" xlink:to="lab_efc_Netincomelossattributabletoparentandparticipatingnoncontrollinginterest" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardVestingPeriodofAwardsGrantedDuringPeriodTableTextBlock_85568a70-d412-4113-88ce-e480557ff366_terseLabel_en-US" xlink:label="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardVestingPeriodofAwardsGrantedDuringPeriodTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unvested LTIP Units</link:label>
    <link:label id="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardVestingPeriodofAwardsGrantedDuringPeriodTableTextBlock_label_en-US" xlink:label="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardVestingPeriodofAwardsGrantedDuringPeriodTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Compensation Arrangement by Share-based Payment Award - Vesting Period of Awards Granted During Period [Table Text Block]</link:label>
    <link:label id="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardVestingPeriodofAwardsGrantedDuringPeriodTableTextBlock_documentation_en-US" xlink:label="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardVestingPeriodofAwardsGrantedDuringPeriodTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Details of unvested LTIP units at the end of the period.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardVestingPeriodofAwardsGrantedDuringPeriodTableTextBlock" xlink:href="efc-20201231.xsd#efc_SharebasedCompensationArrangementbySharebasedPaymentAwardVestingPeriodofAwardsGrantedDuringPeriodTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardVestingPeriodofAwardsGrantedDuringPeriodTableTextBlock" xlink:to="lab_efc_SharebasedCompensationArrangementbySharebasedPaymentAwardVestingPeriodofAwardsGrantedDuringPeriodTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_GainLossOnSecuritizationOfFinancialAssets_1b63f8b3-b875-4869-9c10-df81bd9cd85a_terseLabel_en-US" xlink:label="lab_us-gaap_GainLossOnSecuritizationOfFinancialAssets" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loss on securitization of financial assets</link:label>
    <link:label id="lab_us-gaap_GainLossOnSecuritizationOfFinancialAssets_label_en-US" xlink:label="lab_us-gaap_GainLossOnSecuritizationOfFinancialAssets" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Gain (Loss) on Securitization of Financial Assets</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_GainLossOnSecuritizationOfFinancialAssets" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_GainLossOnSecuritizationOfFinancialAssets"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_GainLossOnSecuritizationOfFinancialAssets" xlink:to="lab_us-gaap_GainLossOnSecuritizationOfFinancialAssets" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Contributionfromnoncontrollinginterestnoncash_7b66af80-3f99-4665-8324-22d127290497_terseLabel_en-US" xlink:label="lab_efc_Contributionfromnoncontrollinginterestnoncash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Contribution from non-controlling interest (non-cash)</link:label>
    <link:label id="lab_efc_Contributionfromnoncontrollinginterestnoncash_label_en-US" xlink:label="lab_efc_Contributionfromnoncontrollinginterestnoncash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Contribution from non-controlling interest (non-cash)</link:label>
    <link:label id="lab_efc_Contributionfromnoncontrollinginterestnoncash_documentation_en-US" xlink:label="lab_efc_Contributionfromnoncontrollinginterestnoncash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Contribution from non-controlling interest (non-cash)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Contributionfromnoncontrollinginterestnoncash" xlink:href="efc-20201231.xsd#efc_Contributionfromnoncontrollinginterestnoncash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Contributionfromnoncontrollinginterestnoncash" xlink:to="lab_efc_Contributionfromnoncontrollinginterestnoncash" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CashAndCashEquivalentsAtCarryingValue_1811bf1d-fc63-4734-9145-7a7e8adb1490_verboseLabel_en-US" xlink:label="lab_us-gaap_CashAndCashEquivalentsAtCarryingValue" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash and cash equivalents</link:label>
    <link:label id="lab_us-gaap_CashAndCashEquivalentsAtCarryingValue_08b841a6-141f-494d-a4bd-b93e30f6550e_terseLabel_en-US" xlink:label="lab_us-gaap_CashAndCashEquivalentsAtCarryingValue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash and Cash Equivalents, at Carrying Value</link:label>
    <link:label id="lab_us-gaap_CashAndCashEquivalentsAtCarryingValue_label_en-US" xlink:label="lab_us-gaap_CashAndCashEquivalentsAtCarryingValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash and Cash Equivalents, at Carrying Value</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CashAndCashEquivalentsAtCarryingValue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CashAndCashEquivalentsAtCarryingValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CashAndCashEquivalentsAtCarryingValue" xlink:to="lab_us-gaap_CashAndCashEquivalentsAtCarryingValue" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PreferredStockAmountOfPreferredDividendsInArrears_9d135172-e3b8-47e7-a4e5-2188d2915bbf_terseLabel_en-US" xlink:label="lab_us-gaap_PreferredStockAmountOfPreferredDividendsInArrears" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Amount of Preferred Dividends in Arrears</link:label>
    <link:label id="lab_us-gaap_PreferredStockAmountOfPreferredDividendsInArrears_label_en-US" xlink:label="lab_us-gaap_PreferredStockAmountOfPreferredDividendsInArrears" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Amount of Preferred Dividends in Arrears</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PreferredStockAmountOfPreferredDividendsInArrears" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PreferredStockAmountOfPreferredDividendsInArrears"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PreferredStockAmountOfPreferredDividendsInArrears" xlink:to="lab_us-gaap_PreferredStockAmountOfPreferredDividendsInArrears" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NoninterestIncomeAbstract_2a86da9b-8175-480e-b431-c76b4e6c25d8_terseLabel_en-US" xlink:label="lab_us-gaap_NoninterestIncomeAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Expenses</link:label>
    <link:label id="lab_us-gaap_NoninterestIncomeAbstract_label_en-US" xlink:label="lab_us-gaap_NoninterestIncomeAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noninterest Income [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NoninterestIncomeAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NoninterestIncomeAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NoninterestIncomeAbstract" xlink:to="lab_us-gaap_NoninterestIncomeAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RealizedInvestmentGainsLossesAbstract_e97df3af-2ee1-4e11-b54b-de8d4fd35455_terseLabel_en-US" xlink:label="lab_us-gaap_RealizedInvestmentGainsLossesAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized Investment Gains (Losses) [Abstract]</link:label>
    <link:label id="lab_us-gaap_RealizedInvestmentGainsLossesAbstract_label_en-US" xlink:label="lab_us-gaap_RealizedInvestmentGainsLossesAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized Investment Gains (Losses) [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RealizedInvestmentGainsLossesAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RealizedInvestmentGainsLossesAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RealizedInvestmentGainsLossesAbstract" xlink:to="lab_us-gaap_RealizedInvestmentGainsLossesAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FinancialInstrumentsSoldNotYetPurchasedAmortizedCost_77919c80-7575-423c-8479-e2f2c3aa7fd8_terseLabel_en-US" xlink:label="lab_efc_FinancialInstrumentsSoldNotYetPurchasedAmortizedCost" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Sold, Not yet Purchased, Sale Proceeds</link:label>
    <link:label id="lab_efc_FinancialInstrumentsSoldNotYetPurchasedAmortizedCost_label_en-US" xlink:label="lab_efc_FinancialInstrumentsSoldNotYetPurchasedAmortizedCost" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instruments Sold, Not Yet Purchased, Amortized Cost</link:label>
    <link:label id="lab_efc_FinancialInstrumentsSoldNotYetPurchasedAmortizedCost_documentation_en-US" xlink:label="lab_efc_FinancialInstrumentsSoldNotYetPurchasedAmortizedCost" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instruments Sold, Not yet Purchased, Amortized Cost</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FinancialInstrumentsSoldNotYetPurchasedAmortizedCost" xlink:href="efc-20201231.xsd#efc_FinancialInstrumentsSoldNotYetPurchasedAmortizedCost"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FinancialInstrumentsSoldNotYetPurchasedAmortizedCost" xlink:to="lab_efc_FinancialInstrumentsSoldNotYetPurchasedAmortizedCost" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12Months_4932679d-0335-4454-8f11-dfd5ec8fbf31_terseLabel_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12Months" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Less than 12 Months</link:label>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12Months_label_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12Months" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Continuous Unrealized Loss Position, Less than 12 Months</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12Months" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12Months"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12Months" xlink:to="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPositionLessThan12Months" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MortgagerelatedResidentialMember_487f5e6f-b6b8-4c16-987e-17a8b9e6a71a_terseLabel_en-US" xlink:label="lab_efc_MortgagerelatedResidentialMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage-related Residential [Member]</link:label>
    <link:label id="lab_efc_MortgagerelatedResidentialMember_label_en-US" xlink:label="lab_efc_MortgagerelatedResidentialMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage-related Residential [Member]</link:label>
    <link:label id="lab_efc_MortgagerelatedResidentialMember_documentation_en-US" xlink:label="lab_efc_MortgagerelatedResidentialMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage-related Residential [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MortgagerelatedResidentialMember" xlink:href="efc-20201231.xsd#efc_MortgagerelatedResidentialMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MortgagerelatedResidentialMember" xlink:to="lab_efc_MortgagerelatedResidentialMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearTwo_52716a69-02d9-4dd6-9c7c-59a5d2c21542_terseLabel_en-US" xlink:label="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearTwo" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Maturities, Repayments of Principal in Rolling Year Two</link:label>
    <link:label id="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearTwo_label_en-US" xlink:label="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearTwo" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Maturities, Repayments of Principal in Rolling Year Two</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearTwo" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearTwo"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearTwo" xlink:to="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearTwo" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonExchangeTradedCommonEquityInvestmentinMortgageRelatedEntitiesMember_774c44f1-41bf-48fc-9309-c8fad70ff133_terseLabel_en-US" xlink:label="lab_efc_NonExchangeTradedCommonEquityInvestmentinMortgageRelatedEntitiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-exchange traded common equity investment in mortgage-related entity</link:label>
    <link:label id="lab_efc_NonExchangeTradedCommonEquityInvestmentinMortgageRelatedEntitiesMember_label_en-US" xlink:label="lab_efc_NonExchangeTradedCommonEquityInvestmentinMortgageRelatedEntitiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Common Equity Investment in Mortgage-Related Entities [Member]</link:label>
    <link:label id="lab_efc_NonExchangeTradedCommonEquityInvestmentinMortgageRelatedEntitiesMember_documentation_en-US" xlink:label="lab_efc_NonExchangeTradedCommonEquityInvestmentinMortgageRelatedEntitiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Common Equity Investment in Mortgage-Related Entities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonExchangeTradedCommonEquityInvestmentinMortgageRelatedEntitiesMember" xlink:href="efc-20201231.xsd#efc_NonExchangeTradedCommonEquityInvestmentinMortgageRelatedEntitiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonExchangeTradedCommonEquityInvestmentinMortgageRelatedEntitiesMember" xlink:to="lab_efc_NonExchangeTradedCommonEquityInvestmentinMortgageRelatedEntitiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StatementEquityComponentsAxis_95eb2777-ea0c-4ee7-95a3-632d801f8e60_terseLabel_en-US" xlink:label="lab_us-gaap_StatementEquityComponentsAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Components [Axis]</link:label>
    <link:label id="lab_us-gaap_StatementEquityComponentsAxis_label_en-US" xlink:label="lab_us-gaap_StatementEquityComponentsAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Components [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StatementEquityComponentsAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StatementEquityComponentsAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StatementEquityComponentsAxis" xlink:to="lab_us-gaap_StatementEquityComponentsAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DeferredTaxLiabilities_64089cea-8233-427f-abb4-32996b07f478_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_DeferredTaxLiabilities" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Tax Liabilities, Net</link:label>
    <link:label id="lab_us-gaap_DeferredTaxLiabilities_label_en-US" xlink:label="lab_us-gaap_DeferredTaxLiabilities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Tax Liabilities, Net</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DeferredTaxLiabilities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DeferredTaxLiabilities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DeferredTaxLiabilities" xlink:to="lab_us-gaap_DeferredTaxLiabilities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleOfShareholdersEquityPerShareTableTextBlock_ba569758-199c-4524-9722-89ff2ac8bf93_terseLabel_en-US" xlink:label="lab_efc_ScheduleOfShareholdersEquityPerShareTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Shareholders Equity Per Share</link:label>
    <link:label id="lab_efc_ScheduleOfShareholdersEquityPerShareTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleOfShareholdersEquityPerShareTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Shareholders Equity Per Share [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleOfShareholdersEquityPerShareTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleOfShareholdersEquityPerShareTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Tabular disclosure of the rollforward of the Company's shareholder's equity per share from the begining of the period to the end of the period.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleOfShareholdersEquityPerShareTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleOfShareholdersEquityPerShareTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleOfShareholdersEquityPerShareTableTextBlock" xlink:to="lab_efc_ScheduleOfShareholdersEquityPerShareTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsAxis_d80088b2-0236-41b4-ade9-b6c07beaf0c0_terseLabel_en-US" xlink:label="lab_us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Restricted Cash [Axis]</link:label>
    <link:label id="lab_us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsAxis_label_en-US" xlink:label="lab_us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Restricted Cash and Cash Equivalents [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsAxis" xlink:to="lab_us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinanceLoansAndLeasesReceivablePolicy_b3c1b7bc-8966-45b5-a831-94c0b32742a5_terseLabel_en-US" xlink:label="lab_us-gaap_FinanceLoansAndLeasesReceivablePolicy" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accounting for Loans</link:label>
    <link:label id="lab_us-gaap_FinanceLoansAndLeasesReceivablePolicy_label_en-US" xlink:label="lab_us-gaap_FinanceLoansAndLeasesReceivablePolicy" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinanceLoansAndLeasesReceivablePolicy" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinanceLoansAndLeasesReceivablePolicy"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinanceLoansAndLeasesReceivablePolicy" xlink:to="lab_us-gaap_FinanceLoansAndLeasesReceivablePolicy" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PaymentsToAcquireEquityMethodInvestments_e345d8ec-8955-4720-8045-6eefe57c48db_negatedLabel_en-US" xlink:label="lab_us-gaap_PaymentsToAcquireEquityMethodInvestments" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Contributions to investments in unconsolidated entities</link:label>
    <link:label id="lab_us-gaap_PaymentsToAcquireEquityMethodInvestments_label_en-US" xlink:label="lab_us-gaap_PaymentsToAcquireEquityMethodInvestments" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments to Acquire Equity Method Investments</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PaymentsToAcquireEquityMethodInvestments" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsToAcquireEquityMethodInvestments"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PaymentsToAcquireEquityMethodInvestments" xlink:to="lab_us-gaap_PaymentsToAcquireEquityMethodInvestments" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EffectiveOwnershipPercentage_eb440130-ac7e-46cf-b625-2d121bc77d22_terseLabel_en-US" xlink:label="lab_efc_EffectiveOwnershipPercentage" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Effective Ownership Percentage</link:label>
    <link:label id="lab_efc_EffectiveOwnershipPercentage_label_en-US" xlink:label="lab_efc_EffectiveOwnershipPercentage" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Effective Ownership Percentage</link:label>
    <link:label id="lab_efc_EffectiveOwnershipPercentage_documentation_en-US" xlink:label="lab_efc_EffectiveOwnershipPercentage" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Effective percentage ownership before the effects of consolidation of entities' respective segregated portions of the VIE.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EffectiveOwnershipPercentage" xlink:href="efc-20201231.xsd#efc_EffectiveOwnershipPercentage"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EffectiveOwnershipPercentage" xlink:to="lab_efc_EffectiveOwnershipPercentage" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_GuaranteeObligationsMaximumExposure_fcf3637e-4b95-4edc-b267-ceb6d6f0b228_terseLabel_en-US" xlink:label="lab_us-gaap_GuaranteeObligationsMaximumExposure" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maximum guarantees</link:label>
    <link:label id="lab_us-gaap_GuaranteeObligationsMaximumExposure_label_en-US" xlink:label="lab_us-gaap_GuaranteeObligationsMaximumExposure" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Guarantor Obligations, Maximum Exposure, Undiscounted</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_GuaranteeObligationsMaximumExposure" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_GuaranteeObligationsMaximumExposure"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_GuaranteeObligationsMaximumExposure" xlink:to="lab_us-gaap_GuaranteeObligationsMaximumExposure" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetValue_00112e32-5b3c-45da-a9e1-2c27f7b37abd_periodStartLabel_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetValue" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets, Beginning Balance</link:label>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetValue_635cc085-9e11-43f2-a840-c99f36054c27_periodEndLabel_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetValue" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets, Ending Balance</link:label>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetValue_label_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement with Unobservable Inputs Reconciliation, Recurring Basis, Asset Value</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetValue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetValue" xlink:to="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetValue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NoteAgreementLendingAmountPerAgreement_91f3afe3-2300-4ba9-a1ec-be7394d4a744_terseLabel_en-US" xlink:label="lab_efc_NoteAgreementLendingAmountPerAgreement" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Note Agreement, Lending Amount Per Agreement</link:label>
    <link:label id="lab_efc_NoteAgreementLendingAmountPerAgreement_label_en-US" xlink:label="lab_efc_NoteAgreementLendingAmountPerAgreement" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Note Agreement, Lending Amount Per Agreement</link:label>
    <link:label id="lab_efc_NoteAgreementLendingAmountPerAgreement_documentation_en-US" xlink:label="lab_efc_NoteAgreementLendingAmountPerAgreement" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maximum amount Company agrees to lend to third party under respective note agreement.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NoteAgreementLendingAmountPerAgreement" xlink:href="efc-20201231.xsd#efc_NoteAgreementLendingAmountPerAgreement"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NoteAgreementLendingAmountPerAgreement" xlink:to="lab_efc_NoteAgreementLendingAmountPerAgreement" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PaymentsForCapitalImprovements_6bc6602f-3952-4a57-ada1-d0d76dbe9198_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_PaymentsForCapitalImprovements" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Capital improvements of real estate owned</link:label>
    <link:label id="lab_us-gaap_PaymentsForCapitalImprovements_label_en-US" xlink:label="lab_us-gaap_PaymentsForCapitalImprovements" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments for Capital Improvements</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PaymentsForCapitalImprovements" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsForCapitalImprovements"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PaymentsForCapitalImprovements" xlink:to="lab_us-gaap_PaymentsForCapitalImprovements" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AccretionAmortizationOfDiscountsAndPremiumsInvestments_a841c9c9-5c79-426e-be2f-0379db47d4c0_negatedLabel_en-US" xlink:label="lab_us-gaap_AccretionAmortizationOfDiscountsAndPremiumsInvestments" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amortization of premiums and accretion of discounts, net</link:label>
    <link:label id="lab_us-gaap_AccretionAmortizationOfDiscountsAndPremiumsInvestments_14f93644-9605-4473-9635-61a101b3a426_terseLabel_en-US" xlink:label="lab_us-gaap_AccretionAmortizationOfDiscountsAndPremiumsInvestments" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Amortization</link:label>
    <link:label id="lab_us-gaap_AccretionAmortizationOfDiscountsAndPremiumsInvestments_label_en-US" xlink:label="lab_us-gaap_AccretionAmortizationOfDiscountsAndPremiumsInvestments" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accretion (Amortization) of Discounts and Premiums, Investments</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AccretionAmortizationOfDiscountsAndPremiumsInvestments" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AccretionAmortizationOfDiscountsAndPremiumsInvestments"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AccretionAmortizationOfDiscountsAndPremiumsInvestments" xlink:to="lab_us-gaap_AccretionAmortizationOfDiscountsAndPremiumsInvestments" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ResidentialMortgageBackedSecuritiesMember_470accc4-5634-4640-874b-6c40b3ec2d47_terseLabel_en-US" xlink:label="lab_us-gaap_ResidentialMortgageBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Agency RMBS</link:label>
    <link:label id="lab_us-gaap_ResidentialMortgageBackedSecuritiesMember_label_en-US" xlink:label="lab_us-gaap_ResidentialMortgageBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Residential Mortgage Backed Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ResidentialMortgageBackedSecuritiesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ResidentialMortgageBackedSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ResidentialMortgageBackedSecuritiesMember" xlink:to="lab_us-gaap_ResidentialMortgageBackedSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionAccumulatedLoss_a3b2931e-f2f8-4afe-8d72-7afaf57b04da_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionAccumulatedLoss" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized Loss, Total</link:label>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionAccumulatedLoss_label_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionAccumulatedLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Unrealized Loss Position, Accumulated Loss</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionAccumulatedLoss" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionAccumulatedLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionAccumulatedLoss" xlink:to="lab_us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPositionAccumulatedLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_BorrowingsMember_96edfd9b-90b9-4550-b1c7-e76f224ff460_terseLabel_en-US" xlink:label="lab_us-gaap_BorrowingsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total</link:label>
    <link:label id="lab_us-gaap_BorrowingsMember_label_en-US" xlink:label="lab_us-gaap_BorrowingsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Borrowings [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_BorrowingsMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_BorrowingsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_BorrowingsMember" xlink:to="lab_us-gaap_BorrowingsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CurrentFederalTaxExpenseBenefit_ade2fecd-d8ee-44a8-9700-4f342df4213a_terseLabel_en-US" xlink:label="lab_us-gaap_CurrentFederalTaxExpenseBenefit" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Current Federal Tax Expense (Benefit)</link:label>
    <link:label id="lab_us-gaap_CurrentFederalTaxExpenseBenefit_label_en-US" xlink:label="lab_us-gaap_CurrentFederalTaxExpenseBenefit" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Current Federal Tax Expense (Benefit)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CurrentFederalTaxExpenseBenefit" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CurrentFederalTaxExpenseBenefit"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CurrentFederalTaxExpenseBenefit" xlink:to="lab_us-gaap_CurrentFederalTaxExpenseBenefit" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleofWarrantContactsAbstract_7f1092d1-0189-4867-9f26-92ca007c4185_terseLabel_en-US" xlink:label="lab_efc_ScheduleofWarrantContactsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Warrant Contacts [Abstract]</link:label>
    <link:label id="lab_efc_ScheduleofWarrantContactsAbstract_label_en-US" xlink:label="lab_efc_ScheduleofWarrantContactsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Warrant Contacts [Abstract]</link:label>
    <link:label id="lab_efc_ScheduleofWarrantContactsAbstract_documentation_en-US" xlink:label="lab_efc_ScheduleofWarrantContactsAbstract" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Warrant Contacts [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleofWarrantContactsAbstract" xlink:href="efc-20201231.xsd#efc_ScheduleofWarrantContactsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleofWarrantContactsAbstract" xlink:to="lab_efc_ScheduleofWarrantContactsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ShortMember_48c22ab3-d8d9-4cf5-a3d6-9802cf34f884_terseLabel_en-US" xlink:label="lab_us-gaap_ShortMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Short</link:label>
    <link:label id="lab_us-gaap_ShortMember_label_en-US" xlink:label="lab_us-gaap_ShortMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Short [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ShortMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ShortMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ShortMember" xlink:to="lab_us-gaap_ShortMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CollateralizedLoanObligationsMember_3b27fac0-7a3c-4193-a387-dc6ba2d24834_terseLabel_en-US" xlink:label="lab_us-gaap_CollateralizedLoanObligationsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLOs</link:label>
    <link:label id="lab_us-gaap_CollateralizedLoanObligationsMember_3cac80d6-3dcb-47a4-b513-f1d0c564c5c2_verboseLabel_en-US" xlink:label="lab_us-gaap_CollateralizedLoanObligationsMember" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLOs</link:label>
    <link:label id="lab_us-gaap_CollateralizedLoanObligationsMember_label_en-US" xlink:label="lab_us-gaap_CollateralizedLoanObligationsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Collateralized Loan Obligations [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CollateralizedLoanObligationsMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CollateralizedLoanObligationsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CollateralizedLoanObligationsMember" xlink:to="lab_us-gaap_CollateralizedLoanObligationsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_IncreaseDecreaseinAccruedIncentiveFee_a7129ad6-6dfa-43c1-9270-2e89a8741536_terseLabel_en-US" xlink:label="lab_efc_IncreaseDecreaseinAccruedIncentiveFee" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Incentive fee payable to affiliate</link:label>
    <link:label id="lab_efc_IncreaseDecreaseinAccruedIncentiveFee_label_en-US" xlink:label="lab_efc_IncreaseDecreaseinAccruedIncentiveFee" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (Decrease) in Accrued Incentive Fee</link:label>
    <link:label id="lab_efc_IncreaseDecreaseinAccruedIncentiveFee_documentation_en-US" xlink:label="lab_efc_IncreaseDecreaseinAccruedIncentiveFee" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Then increase (decrease) during the reporting period in the amount payable to the Manager relating to performance based fees.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_IncreaseDecreaseinAccruedIncentiveFee" xlink:href="efc-20201231.xsd#efc_IncreaseDecreaseinAccruedIncentiveFee"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_IncreaseDecreaseinAccruedIncentiveFee" xlink:to="lab_efc_IncreaseDecreaseinAccruedIncentiveFee" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MortgageLoansInProcessOfForeclosureAmount_2c88c501-b9fc-4ffc-8600-8d4791e1b02d_terseLabel_en-US" xlink:label="lab_us-gaap_MortgageLoansInProcessOfForeclosureAmount" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage Loans in Process of Foreclosure, Amount</link:label>
    <link:label id="lab_us-gaap_MortgageLoansInProcessOfForeclosureAmount_label_en-US" xlink:label="lab_us-gaap_MortgageLoansInProcessOfForeclosureAmount" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage Loans in Process of Foreclosure, Amount</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MortgageLoansInProcessOfForeclosureAmount" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MortgageLoansInProcessOfForeclosureAmount"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MortgageLoansInProcessOfForeclosureAmount" xlink:to="lab_us-gaap_MortgageLoansInProcessOfForeclosureAmount" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Administrationfeeexpense_c2958c76-7d4b-4f5b-9671-2e92addc7c69_terseLabel_en-US" xlink:label="lab_efc_Administrationfeeexpense" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Administration fee expense</link:label>
    <link:label id="lab_efc_Administrationfeeexpense_label_en-US" xlink:label="lab_efc_Administrationfeeexpense" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Administration fee expense</link:label>
    <link:label id="lab_efc_Administrationfeeexpense_documentation_en-US" xlink:label="lab_efc_Administrationfeeexpense" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fee paid to third party for outsourced accounting and record keeping functions.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Administrationfeeexpense" xlink:href="efc-20201231.xsd#efc_Administrationfeeexpense"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Administrationfeeexpense" xlink:to="lab_efc_Administrationfeeexpense" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleofUnvestedLTIPUnitsLineItems_ef40a30b-2905-46a6-b552-64a9ff48a718_terseLabel_en-US" xlink:label="lab_efc_ScheduleofUnvestedLTIPUnitsLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Unvested LTIP Units [Line Items]</link:label>
    <link:label id="lab_efc_ScheduleofUnvestedLTIPUnitsLineItems_label_en-US" xlink:label="lab_efc_ScheduleofUnvestedLTIPUnitsLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Unvested LTIP Units [Line Items]</link:label>
    <link:label id="lab_efc_ScheduleofUnvestedLTIPUnitsLineItems_documentation_en-US" xlink:label="lab_efc_ScheduleofUnvestedLTIPUnitsLineItems" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Line Items] for Schedule of Unvested LTIP Units [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleofUnvestedLTIPUnitsLineItems" xlink:href="efc-20201231.xsd#efc_ScheduleofUnvestedLTIPUnitsLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleofUnvestedLTIPUnitsLineItems" xlink:to="lab_efc_ScheduleofUnvestedLTIPUnitsLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CorporateDebtandEquityMember_6dde2d11-09c4-453c-b527-b318f093e2ab_terseLabel_en-US" xlink:label="lab_efc_CorporateDebtandEquityMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate debt and equity</link:label>
    <link:label id="lab_efc_CorporateDebtandEquityMember_label_en-US" xlink:label="lab_efc_CorporateDebtandEquityMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate Debt and Equity [Member]</link:label>
    <link:label id="lab_efc_CorporateDebtandEquityMember_documentation_en-US" xlink:label="lab_efc_CorporateDebtandEquityMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate Debt and Equity [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CorporateDebtandEquityMember" xlink:href="efc-20201231.xsd#efc_CorporateDebtandEquityMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CorporateDebtandEquityMember" xlink:to="lab_efc_CorporateDebtandEquityMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RealEstateOwnedPolicyTextBlock_cc0318c4-54fb-4a76-8bc5-32a6dce80aa7_terseLabel_en-US" xlink:label="lab_efc_RealEstateOwnedPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">REO</link:label>
    <link:label id="lab_efc_RealEstateOwnedPolicyTextBlock_label_en-US" xlink:label="lab_efc_RealEstateOwnedPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Owned [Policy Text Block]</link:label>
    <link:label id="lab_efc_RealEstateOwnedPolicyTextBlock_documentation_en-US" xlink:label="lab_efc_RealEstateOwnedPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The Company's accounting policy for real estate owned.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RealEstateOwnedPolicyTextBlock" xlink:href="efc-20201231.xsd#efc_RealEstateOwnedPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RealEstateOwnedPolicyTextBlock" xlink:to="lab_efc_RealEstateOwnedPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MinorityInterestOwnershipPercentageByNoncontrollingOwners_5892f29c-c0d9-4716-a786-8b1624e70863_terseLabel_en-US" xlink:label="lab_us-gaap_MinorityInterestOwnershipPercentageByNoncontrollingOwners" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ownership Percentage</link:label>
    <link:label id="lab_us-gaap_MinorityInterestOwnershipPercentageByNoncontrollingOwners_label_en-US" xlink:label="lab_us-gaap_MinorityInterestOwnershipPercentageByNoncontrollingOwners" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noncontrolling Interest, Ownership Percentage by Noncontrolling Owners</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MinorityInterestOwnershipPercentageByNoncontrollingOwners" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MinorityInterestOwnershipPercentageByNoncontrollingOwners"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MinorityInterestOwnershipPercentageByNoncontrollingOwners" xlink:to="lab_us-gaap_MinorityInterestOwnershipPercentageByNoncontrollingOwners" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonQMloansecuritizationMember_a356eaa5-3907-4cd0-b49e-3f59816fbc67_terseLabel_en-US" xlink:label="lab_efc_NonQMloansecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-QM loan securitization</link:label>
    <link:label id="lab_efc_NonQMloansecuritizationMember_label_en-US" xlink:label="lab_efc_NonQMloansecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-QM loan securitization [Member]</link:label>
    <link:label id="lab_efc_NonQMloansecuritizationMember_documentation_en-US" xlink:label="lab_efc_NonQMloansecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-QM loan securitization [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonQMloansecuritizationMember" xlink:href="efc-20201231.xsd#efc_NonQMloansecuritizationMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonQMloansecuritizationMember" xlink:to="lab_efc_NonQMloansecuritizationMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NoncontrollinginterestAxis_29312054-bdf7-4966-aed4-fce712f2728b_terseLabel_en-US" xlink:label="lab_efc_NoncontrollinginterestAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling interest [Axis]</link:label>
    <link:label id="lab_efc_NoncontrollinginterestAxis_label_en-US" xlink:label="lab_efc_NoncontrollinginterestAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling interest [Axis]</link:label>
    <link:label id="lab_efc_NoncontrollinginterestAxis_documentation_en-US" xlink:label="lab_efc_NoncontrollinginterestAxis" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling interest [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NoncontrollinginterestAxis" xlink:href="efc-20201231.xsd#efc_NoncontrollinginterestAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NoncontrollinginterestAxis" xlink:to="lab_efc_NoncontrollinginterestAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CashAndCashEquivalentsFairValueDisclosure_7cc83507-c317-47cf-ae12-dc0d891a9136_terseLabel_en-US" xlink:label="lab_us-gaap_CashAndCashEquivalentsFairValueDisclosure" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash and cash equivalents</link:label>
    <link:label id="lab_us-gaap_CashAndCashEquivalentsFairValueDisclosure_label_en-US" xlink:label="lab_us-gaap_CashAndCashEquivalentsFairValueDisclosure" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash and Cash Equivalents, Fair Value Disclosure</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CashAndCashEquivalentsFairValueDisclosure" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CashAndCashEquivalentsFairValueDisclosure"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CashAndCashEquivalentsFairValueDisclosure" xlink:to="lab_us-gaap_CashAndCashEquivalentsFairValueDisclosure" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PaymentsForSecuritiesPurchasedUnderAgreementsToResell_d718629f-29ed-402e-ac28-703d22118cb7_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_PaymentsForSecuritiesPurchasedUnderAgreementsToResell" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments made on reverse repurchase agreements</link:label>
    <link:label id="lab_us-gaap_PaymentsForSecuritiesPurchasedUnderAgreementsToResell_label_en-US" xlink:label="lab_us-gaap_PaymentsForSecuritiesPurchasedUnderAgreementsToResell" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments for Securities Purchased under Agreements to Resell</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PaymentsForSecuritiesPurchasedUnderAgreementsToResell" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsForSecuritiesPurchasedUnderAgreementsToResell"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PaymentsForSecuritiesPurchasedUnderAgreementsToResell" xlink:to="lab_us-gaap_PaymentsForSecuritiesPurchasedUnderAgreementsToResell" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ClosingDateofSecuritizationTransactionAxis_8e159330-c5a4-4449-ae15-1dfedce69105_terseLabel_en-US" xlink:label="lab_efc_ClosingDateofSecuritizationTransactionAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Closing Date of Securitization Transaction [Axis]</link:label>
    <link:label id="lab_efc_ClosingDateofSecuritizationTransactionAxis_label_en-US" xlink:label="lab_efc_ClosingDateofSecuritizationTransactionAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Closing Date of Securitization Transaction [Axis]</link:label>
    <link:label id="lab_efc_ClosingDateofSecuritizationTransactionAxis_documentation_en-US" xlink:label="lab_efc_ClosingDateofSecuritizationTransactionAxis" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Closing Date of Securitization Transaction [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ClosingDateofSecuritizationTransactionAxis" xlink:href="efc-20201231.xsd#efc_ClosingDateofSecuritizationTransactionAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ClosingDateofSecuritizationTransactionAxis" xlink:to="lab_efc_ClosingDateofSecuritizationTransactionAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DividendsPolicyTextBlock_8784170d-7acc-4813-91cf-1022b5a35b97_terseLabel_en-US" xlink:label="lab_efc_DividendsPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends</link:label>
    <link:label id="lab_efc_DividendsPolicyTextBlock_label_en-US" xlink:label="lab_efc_DividendsPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends [Policy Text Block]</link:label>
    <link:label id="lab_efc_DividendsPolicyTextBlock_documentation_en-US" xlink:label="lab_efc_DividendsPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disclosure of the Company's accounting policy regarding when dividends payable are recorded.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DividendsPolicyTextBlock" xlink:href="efc-20201231.xsd#efc_DividendsPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DividendsPolicyTextBlock" xlink:to="lab_efc_DividendsPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancingReceivablesEqualToGreaterThan90DaysPastDueMember_5d5cdada-c28f-4614-8126-4169d40bae09_terseLabel_en-US" xlink:label="lab_us-gaap_FinancingReceivablesEqualToGreaterThan90DaysPastDueMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">90 Days or More Past Due [Member]</link:label>
    <link:label id="lab_us-gaap_FinancingReceivablesEqualToGreaterThan90DaysPastDueMember_label_en-US" xlink:label="lab_us-gaap_FinancingReceivablesEqualToGreaterThan90DaysPastDueMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Asset, Equal to or Greater than 90 Days Past Due [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancingReceivablesEqualToGreaterThan90DaysPastDueMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancingReceivablesEqualToGreaterThan90DaysPastDueMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancingReceivablesEqualToGreaterThan90DaysPastDueMember" xlink:to="lab_us-gaap_FinancingReceivablesEqualToGreaterThan90DaysPastDueMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DisposalGroupClassificationDomain_ccac8ab7-f365-4fdd-ae7a-8225b30ced2b_terseLabel_en-US" xlink:label="lab_us-gaap_DisposalGroupClassificationDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disposal Group Classification [Domain]</link:label>
    <link:label id="lab_us-gaap_DisposalGroupClassificationDomain_label_en-US" xlink:label="lab_us-gaap_DisposalGroupClassificationDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disposal Group Classification [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DisposalGroupClassificationDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DisposalGroupClassificationDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DisposalGroupClassificationDomain" xlink:to="lab_us-gaap_DisposalGroupClassificationDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Securitizationpricingdate_5feb5015-f567-436c-8265-21fa76dff360_terseLabel_en-US" xlink:label="lab_efc_Securitizationpricingdate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLO Pricing Date</link:label>
    <link:label id="lab_efc_Securitizationpricingdate_label_en-US" xlink:label="lab_efc_Securitizationpricingdate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization pricing date</link:label>
    <link:label id="lab_efc_Securitizationpricingdate_documentation_en-US" xlink:label="lab_efc_Securitizationpricingdate" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization pricing date (trade date).</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Securitizationpricingdate" xlink:href="efc-20201231.xsd#efc_Securitizationpricingdate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Securitizationpricingdate" xlink:to="lab_efc_Securitizationpricingdate" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_U.S.TreasuryNote2.75PercentMaturityDate202304Member_01f7d3f2-bb0b-480a-acf5-0875cf0bf46a_terseLabel_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.75PercentMaturityDate202304Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.75 Percent, Maturity Date 2023 04 [Member]</link:label>
    <link:label id="lab_efc_U.S.TreasuryNote2.75PercentMaturityDate202304Member_label_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.75PercentMaturityDate202304Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.75 Percent, Maturity Date 2023 04 [Member]</link:label>
    <link:label id="lab_efc_U.S.TreasuryNote2.75PercentMaturityDate202304Member_documentation_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.75PercentMaturityDate202304Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.75 Percent, Maturity Date 2023 04 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_U.S.TreasuryNote2.75PercentMaturityDate202304Member" xlink:href="efc-20201231.xsd#efc_U.S.TreasuryNote2.75PercentMaturityDate202304Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_U.S.TreasuryNote2.75PercentMaturityDate202304Member" xlink:to="lab_efc_U.S.TreasuryNote2.75PercentMaturityDate202304Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_IncentiveFeeRate_211150a5-550a-4572-a996-7e2599a1f14f_terseLabel_en-US" xlink:label="lab_efc_IncentiveFeeRate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Incentive fee rate</link:label>
    <link:label id="lab_efc_IncentiveFeeRate_label_en-US" xlink:label="lab_efc_IncentiveFeeRate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Incentive fee rate</link:label>
    <link:label id="lab_efc_IncentiveFeeRate_documentation_en-US" xlink:label="lab_efc_IncentiveFeeRate" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The rate at which incentive fees are charged (when applicable), per the management agreement.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_IncentiveFeeRate" xlink:href="efc-20201231.xsd#efc_IncentiveFeeRate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_IncentiveFeeRate" xlink:to="lab_efc_IncentiveFeeRate" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RelatedPartyTransactionAxis_5a764b39-3630-4a96-8710-1ee375e3535d_terseLabel_en-US" xlink:label="lab_us-gaap_RelatedPartyTransactionAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party Transaction [Axis]</link:label>
    <link:label id="lab_us-gaap_RelatedPartyTransactionAxis_label_en-US" xlink:label="lab_us-gaap_RelatedPartyTransactionAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party Transaction [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RelatedPartyTransactionAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RelatedPartyTransactionAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RelatedPartyTransactionAxis" xlink:to="lab_us-gaap_RelatedPartyTransactionAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_IncreaseDecreaseInReceivableForSecuritiesSold_32542ab8-b3d5-43f2-a06d-d1d2990c6165_negatedTerseLabel_en-US" xlink:label="lab_efc_IncreaseDecreaseInReceivableForSecuritiesSold" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Receivable for securities sold and financial derivatives</link:label>
    <link:label id="lab_efc_IncreaseDecreaseInReceivableForSecuritiesSold_label_en-US" xlink:label="lab_efc_IncreaseDecreaseInReceivableForSecuritiesSold" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase decrease in receivable for securities sold</link:label>
    <link:label id="lab_efc_IncreaseDecreaseInReceivableForSecuritiesSold_documentation_en-US" xlink:label="lab_efc_IncreaseDecreaseInReceivableForSecuritiesSold" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The increase (decrease) in the amount of receivables due from securities sold prior to period end where the transactions have not yet settled.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_IncreaseDecreaseInReceivableForSecuritiesSold" xlink:href="efc-20201231.xsd#efc_IncreaseDecreaseInReceivableForSecuritiesSold"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_IncreaseDecreaseInReceivableForSecuritiesSold" xlink:to="lab_efc_IncreaseDecreaseInReceivableForSecuritiesSold" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NetCashProvidedByUsedInOperatingActivities_7af9ec5f-3c01-4c2d-9b9b-a7ca10a9f85d_terseLabel_en-US" xlink:label="lab_us-gaap_NetCashProvidedByUsedInOperatingActivities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net cash provided by (used in) operating activities</link:label>
    <link:label id="lab_us-gaap_NetCashProvidedByUsedInOperatingActivities_label_en-US" xlink:label="lab_us-gaap_NetCashProvidedByUsedInOperatingActivities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Cash Provided by (Used in) Operating Activities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NetCashProvidedByUsedInOperatingActivities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NetCashProvidedByUsedInOperatingActivities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NetCashProvidedByUsedInOperatingActivities" xlink:to="lab_us-gaap_NetCashProvidedByUsedInOperatingActivities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleOfExpenseRatiosToAverageEquityTableTextBlock_f281d79e-db26-4cd4-882e-240a268b86a4_terseLabel_en-US" xlink:label="lab_efc_ScheduleOfExpenseRatiosToAverageEquityTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Expense Ratios to Average Equity</link:label>
    <link:label id="lab_efc_ScheduleOfExpenseRatiosToAverageEquityTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleOfExpenseRatiosToAverageEquityTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Expense Ratios to Average Equity [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleOfExpenseRatiosToAverageEquityTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleOfExpenseRatiosToAverageEquityTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Expense ratios for the period using average total equity (including non-controlling interest).</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleOfExpenseRatiosToAverageEquityTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleOfExpenseRatiosToAverageEquityTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleOfExpenseRatiosToAverageEquityTableTextBlock" xlink:to="lab_efc_ScheduleOfExpenseRatiosToAverageEquityTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfAvailableForSaleSecuritiesReconciliationTableTextBlock_3031ba50-e805-4f03-b7af-a706a963e7ce_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfAvailableForSaleSecuritiesReconciliationTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Summary of Investment Holdings</link:label>
    <link:label id="lab_us-gaap_ScheduleOfAvailableForSaleSecuritiesReconciliationTableTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfAvailableForSaleSecuritiesReconciliationTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Available-for-sale Securities Reconciliation [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfAvailableForSaleSecuritiesReconciliationTableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfAvailableForSaleSecuritiesReconciliationTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfAvailableForSaleSecuritiesReconciliationTableTextBlock" xlink:to="lab_us-gaap_ScheduleOfAvailableForSaleSecuritiesReconciliationTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_June2019Member_f1777095-32c0-4760-abb2-6a00391d52b7_terseLabel_en-US" xlink:label="lab_efc_June2019Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">June 2019 [Member]</link:label>
    <link:label id="lab_efc_June2019Member_label_en-US" xlink:label="lab_efc_June2019Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">June 2019 [Member]</link:label>
    <link:label id="lab_efc_June2019Member_documentation_en-US" xlink:label="lab_efc_June2019Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">June 2019 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_June2019Member" xlink:href="efc-20201231.xsd#efc_June2019Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_June2019Member" xlink:to="lab_efc_June2019Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_November2020Member_5f0d5a20-6556-4a6e-bb78-3a27cf19fae2_terseLabel_en-US" xlink:label="lab_efc_November2020Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">November 2020</link:label>
    <link:label id="lab_efc_November2020Member_label_en-US" xlink:label="lab_efc_November2020Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">November 2020 [Member]</link:label>
    <link:label id="lab_efc_November2020Member_documentation_en-US" xlink:label="lab_efc_November2020Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">November 2020</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_November2020Member" xlink:href="efc-20201231.xsd#efc_November2020Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_November2020Member" xlink:to="lab_efc_November2020Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InterestIncomeExpenseNet_99cffc60-435c-429d-b75d-adf1d5ab6b7b_totalLabel_en-US" xlink:label="lab_us-gaap_InterestIncomeExpenseNet" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total net interest income</link:label>
    <link:label id="lab_us-gaap_InterestIncomeExpenseNet_f427bac4-2b08-4452-a652-d94b792607ee_terseLabel_en-US" xlink:label="lab_us-gaap_InterestIncomeExpenseNet" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Income (Expense), Net</link:label>
    <link:label id="lab_us-gaap_InterestIncomeExpenseNet_label_en-US" xlink:label="lab_us-gaap_InterestIncomeExpenseNet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Income (Expense), Net</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InterestIncomeExpenseNet" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InterestIncomeExpenseNet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InterestIncomeExpenseNet" xlink:to="lab_us-gaap_InterestIncomeExpenseNet" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MortgageLoansOnRealEstateNewMortgageLoans_735266ba-453e-4283-a3b1-5303dfec0943_terseLabel_en-US" xlink:label="lab_us-gaap_MortgageLoansOnRealEstateNewMortgageLoans" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, New Mortgage Loan</link:label>
    <link:label id="lab_us-gaap_MortgageLoansOnRealEstateNewMortgageLoans_label_en-US" xlink:label="lab_us-gaap_MortgageLoansOnRealEstateNewMortgageLoans" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, New Mortgage Loan</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MortgageLoansOnRealEstateNewMortgageLoans" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MortgageLoansOnRealEstateNewMortgageLoans"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MortgageLoansOnRealEstateNewMortgageLoans" xlink:to="lab_us-gaap_MortgageLoansOnRealEstateNewMortgageLoans" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_UnratedButAgencyGuaranteedMember_d354294b-455e-42a8-abdb-deea7b7f0d5c_terseLabel_en-US" xlink:label="lab_efc_UnratedButAgencyGuaranteedMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrated But Agency Guaranteed [Member]</link:label>
    <link:label id="lab_efc_UnratedButAgencyGuaranteedMember_label_en-US" xlink:label="lab_efc_UnratedButAgencyGuaranteedMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrated But Agency Guaranteed [Member]</link:label>
    <link:label id="lab_efc_UnratedButAgencyGuaranteedMember_documentation_en-US" xlink:label="lab_efc_UnratedButAgencyGuaranteedMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrated But Agency-Guaranteed [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_UnratedButAgencyGuaranteedMember" xlink:href="efc-20201231.xsd#efc_UnratedButAgencyGuaranteedMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_UnratedButAgencyGuaranteedMember" xlink:to="lab_efc_UnratedButAgencyGuaranteedMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ShareholdersEquityPerShareUponConversionOfAllConvertibleUnits_bd7e1605-ecab-4b6d-b1d9-0091a524ac29_terseLabel_en-US" xlink:label="lab_efc_ShareholdersEquityPerShareUponConversionOfAllConvertibleUnits" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Shareholders Equity Per Share Upon Conversion Of All Convertible Units</link:label>
    <link:label id="lab_efc_ShareholdersEquityPerShareUponConversionOfAllConvertibleUnits_label_en-US" xlink:label="lab_efc_ShareholdersEquityPerShareUponConversionOfAllConvertibleUnits" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Shareholders Equity Per Share Upon Conversion Of All Convertible Units</link:label>
    <link:label id="lab_efc_ShareholdersEquityPerShareUponConversionOfAllConvertibleUnits_documentation_en-US" xlink:label="lab_efc_ShareholdersEquityPerShareUponConversionOfAllConvertibleUnits" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The Company's equity per share if all the long term incentive plan units and Operating Partnership units (owned by the non-controlling interest) that have been previously been issued were to be fully vested and exchanged for common shares as of the reporting date.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ShareholdersEquityPerShareUponConversionOfAllConvertibleUnits" xlink:href="efc-20201231.xsd#efc_ShareholdersEquityPerShareUponConversionOfAllConvertibleUnits"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ShareholdersEquityPerShareUponConversionOfAllConvertibleUnits" xlink:to="lab_efc_ShareholdersEquityPerShareUponConversionOfAllConvertibleUnits" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NotesIssued1_c184f4b9-43b3-413b-8e5c-8bc8e55fc6ae_terseLabel_en-US" xlink:label="lab_us-gaap_NotesIssued1" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Issuance of senior notes (non-cash)</link:label>
    <link:label id="lab_us-gaap_NotesIssued1_label_en-US" xlink:label="lab_us-gaap_NotesIssued1" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Notes Issued</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NotesIssued1" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NotesIssued1"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NotesIssued1" xlink:to="lab_us-gaap_NotesIssued1" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_StockRepurchasesAveragePricePerShare_907f9e5e-4e65-49b7-aa93-45cf48cedcb2_terseLabel_en-US" xlink:label="lab_efc_StockRepurchasesAveragePricePerShare" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Average price per share (USD per share)</link:label>
    <link:label id="lab_efc_StockRepurchasesAveragePricePerShare_label_en-US" xlink:label="lab_efc_StockRepurchasesAveragePricePerShare" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stock Repurchases, Average Price Per Share</link:label>
    <link:label id="lab_efc_StockRepurchasesAveragePricePerShare_documentation_en-US" xlink:label="lab_efc_StockRepurchasesAveragePricePerShare" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Average price paid per share repurchased under the Company's share repurchase program.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_StockRepurchasesAveragePricePerShare" xlink:href="efc-20201231.xsd#efc_StockRepurchasesAveragePricePerShare"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_StockRepurchasesAveragePricePerShare" xlink:to="lab_efc_StockRepurchasesAveragePricePerShare" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuredNotesMember_df9a17ff-6172-42d3-b2ed-06b4646e788d_terseLabel_en-US" xlink:label="lab_efc_SecuredNotesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Secured notes</link:label>
    <link:label id="lab_efc_SecuredNotesMember_label_en-US" xlink:label="lab_efc_SecuredNotesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Secured Notes [Member]</link:label>
    <link:label id="lab_efc_SecuredNotesMember_documentation_en-US" xlink:label="lab_efc_SecuredNotesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Secured Notes [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuredNotesMember" xlink:href="efc-20201231.xsd#efc_SecuredNotesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuredNotesMember" xlink:to="lab_efc_SecuredNotesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Collateralonrepurchaseagreementsunsettledsales_248fa775-98cc-4d8d-a42c-c4c4632735f2_terseLabel_en-US" xlink:label="lab_efc_Collateralonrepurchaseagreementsunsettledsales" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Collateral on repurchase agreements, unsettled sales</link:label>
    <link:label id="lab_efc_Collateralonrepurchaseagreementsunsettledsales_label_en-US" xlink:label="lab_efc_Collateralonrepurchaseagreementsunsettledsales" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Collateral on repurchase agreements, unsettled sales</link:label>
    <link:label id="lab_efc_Collateralonrepurchaseagreementsunsettledsales_documentation_en-US" xlink:label="lab_efc_Collateralonrepurchaseagreementsunsettledsales" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The carrying amount as of the date of the latest financial statement presented of securities which are owned but transferred to serve as collateral for repurchase agreements which have been sold prior to period end but remain unsettled at the balance sheet date.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Collateralonrepurchaseagreementsunsettledsales" xlink:href="efc-20201231.xsd#efc_Collateralonrepurchaseagreementsunsettledsales"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Collateralonrepurchaseagreementsunsettledsales" xlink:to="lab_efc_Collateralonrepurchaseagreementsunsettledsales" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_TBASecuritiesSaleContractsMember_61070b32-c88c-473b-84e0-70324f80c0a6_terseLabel_en-US" xlink:label="lab_efc_TBASecuritiesSaleContractsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">TBA Securities, Sale Contracts [Member]</link:label>
    <link:label id="lab_efc_TBASecuritiesSaleContractsMember_label_en-US" xlink:label="lab_efc_TBASecuritiesSaleContractsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">TBA Securities, Sale Contracts [Member]</link:label>
    <link:label id="lab_efc_TBASecuritiesSaleContractsMember_documentation_en-US" xlink:label="lab_efc_TBASecuritiesSaleContractsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Forward contract for the sale (short position) of Agency RMBS in the To-be-announced RMBS market.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_TBASecuritiesSaleContractsMember" xlink:href="efc-20201231.xsd#efc_TBASecuritiesSaleContractsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_TBASecuritiesSaleContractsMember" xlink:to="lab_efc_TBASecuritiesSaleContractsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationFourPointSevenFivePercentMember_4f629ea7-2b12-4b67-b975-ed319cd5c203_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationFourPointSevenFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Four Point Seven Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationFourPointSevenFivePercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationFourPointSevenFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Four Point Seven Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationFourPointSevenFivePercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationFourPointSevenFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Four Point Seven Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationFourPointSevenFivePercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationFourPointSevenFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationFourPointSevenFivePercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationFourPointSevenFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ManagementFeeDescription_bbd377cb-fa40-401e-ba6d-855e492bee1d_terseLabel_en-US" xlink:label="lab_us-gaap_ManagementFeeDescription" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Management Fee, description</link:label>
    <link:label id="lab_us-gaap_ManagementFeeDescription_label_en-US" xlink:label="lab_us-gaap_ManagementFeeDescription" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Management Fee, Description</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ManagementFeeDescription" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ManagementFeeDescription"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ManagementFeeDescription" xlink:to="lab_us-gaap_ManagementFeeDescription" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DeutscheBankSecuritiesMember_26ba7a25-ee66-4347-8343-14ca61348aec_terseLabel_en-US" xlink:label="lab_efc_DeutscheBankSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deutsche Bank Securities [Member]</link:label>
    <link:label id="lab_efc_DeutscheBankSecuritiesMember_label_en-US" xlink:label="lab_efc_DeutscheBankSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deutsche Bank Securities [Member]</link:label>
    <link:label id="lab_efc_DeutscheBankSecuritiesMember_documentation_en-US" xlink:label="lab_efc_DeutscheBankSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deutsche Bank Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DeutscheBankSecuritiesMember" xlink:href="efc-20201231.xsd#efc_DeutscheBankSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DeutscheBankSecuritiesMember" xlink:to="lab_efc_DeutscheBankSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Numberofdaystodeterminenonperformanceofloan_380b7505-c8ca-41ba-99bf-6e544b7f289c_terseLabel_en-US" xlink:label="lab_efc_Numberofdaystodeterminenonperformanceofloan" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of days to determine non-performance of loan</link:label>
    <link:label id="lab_efc_Numberofdaystodeterminenonperformanceofloan_label_en-US" xlink:label="lab_efc_Numberofdaystodeterminenonperformanceofloan" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of days to determine non-performance of loan</link:label>
    <link:label id="lab_efc_Numberofdaystodeterminenonperformanceofloan_documentation_en-US" xlink:label="lab_efc_Numberofdaystodeterminenonperformanceofloan" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The number of days principal and/or interest payments on a loan are delinquent before the Company classifies the loan as "non-performing."</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Numberofdaystodeterminenonperformanceofloan" xlink:href="efc-20201231.xsd#efc_Numberofdaystodeterminenonperformanceofloan"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Numberofdaystodeterminenonperformanceofloan" xlink:to="lab_efc_Numberofdaystodeterminenonperformanceofloan" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Companyspercentageownershipofjointlyownedentity_9b716b2b-f40e-4afe-a864-a48e9892ee66_terseLabel_en-US" xlink:label="lab_efc_Companyspercentageownershipofjointlyownedentity" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Company's current percentage ownership of jointly owned entity</link:label>
    <link:label id="lab_efc_Companyspercentageownershipofjointlyownedentity_label_en-US" xlink:label="lab_efc_Companyspercentageownershipofjointlyownedentity" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Company's percentage ownership of jointly owned entity</link:label>
    <link:label id="lab_efc_Companyspercentageownershipofjointlyownedentity_documentation_en-US" xlink:label="lab_efc_Companyspercentageownershipofjointlyownedentity" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Company's percentage ownership of jointly owned entity with affiliated co-participant.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Companyspercentageownershipofjointlyownedentity" xlink:href="efc-20201231.xsd#efc_Companyspercentageownershipofjointlyownedentity"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Companyspercentageownershipofjointlyownedentity" xlink:to="lab_efc_Companyspercentageownershipofjointlyownedentity" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsMember_2cfbd02a-4a6a-434e-ab84-5bca0524aac4_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationPoolsMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationPoolsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationPoolsMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationPoolsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DividendsPayableDateDeclaredDayMonthAndYear_46c91c9e-521e-46ec-80df-6be457fbfd5d_terseLabel_en-US" xlink:label="lab_us-gaap_DividendsPayableDateDeclaredDayMonthAndYear" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Declared date</link:label>
    <link:label id="lab_us-gaap_DividendsPayableDateDeclaredDayMonthAndYear_label_en-US" xlink:label="lab_us-gaap_DividendsPayableDateDeclaredDayMonthAndYear" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends Payable, Date Declared</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DividendsPayableDateDeclaredDayMonthAndYear" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DividendsPayableDateDeclaredDayMonthAndYear"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DividendsPayableDateDeclaredDayMonthAndYear" xlink:to="lab_us-gaap_DividendsPayableDateDeclaredDayMonthAndYear" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RelatedPartyConsumerLoansTitledinNameofRelatedPartyMember_1c05a652-1537-49ae-8dd8-8df159066026_terseLabel_en-US" xlink:label="lab_efc_RelatedPartyConsumerLoansTitledinNameofRelatedPartyMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party-Consumer Loans Titled in Name of Related Party</link:label>
    <link:label id="lab_efc_RelatedPartyConsumerLoansTitledinNameofRelatedPartyMember_label_en-US" xlink:label="lab_efc_RelatedPartyConsumerLoansTitledinNameofRelatedPartyMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party-Consumer Loans Titled in Name of Related Party [Member]</link:label>
    <link:label id="lab_efc_RelatedPartyConsumerLoansTitledinNameofRelatedPartyMember_documentation_en-US" xlink:label="lab_efc_RelatedPartyConsumerLoansTitledinNameofRelatedPartyMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party-Consumer Loans Titled in Name of Related Party [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RelatedPartyConsumerLoansTitledinNameofRelatedPartyMember" xlink:href="efc-20201231.xsd#efc_RelatedPartyConsumerLoansTitledinNameofRelatedPartyMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RelatedPartyConsumerLoansTitledinNameofRelatedPartyMember" xlink:to="lab_efc_RelatedPartyConsumerLoansTitledinNameofRelatedPartyMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncomeStatementAbstract_7a44b242-c004-4feb-ae48-01a7a82d2c36_terseLabel_en-US" xlink:label="lab_us-gaap_IncomeStatementAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net increase (decrease) in shareholders' equity resulting from operations</link:label>
    <link:label id="lab_us-gaap_IncomeStatementAbstract_label_en-US" xlink:label="lab_us-gaap_IncomeStatementAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Income Statement [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncomeStatementAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncomeStatementAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncomeStatementAbstract" xlink:to="lab_us-gaap_IncomeStatementAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellAbstract_f05c7235-2dc9-4161-8c88-54121f22f362_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reverse repurchase agreements</link:label>
    <link:label id="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellAbstract_label_en-US" xlink:label="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Purchased under Agreements to Resell [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesPurchasedUnderAgreementsToResellAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesPurchasedUnderAgreementsToResellAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesPurchasedUnderAgreementsToResellAbstract" xlink:to="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MeasurementInputEquityPricetoBookMember_f2732662-f630-43c8-912f-1858f8ce30ba_terseLabel_en-US" xlink:label="lab_efc_MeasurementInputEquityPricetoBookMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, Equity Price-to-Book</link:label>
    <link:label id="lab_efc_MeasurementInputEquityPricetoBookMember_label_en-US" xlink:label="lab_efc_MeasurementInputEquityPricetoBookMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, Equity Price-to-Book [Member]</link:label>
    <link:label id="lab_efc_MeasurementInputEquityPricetoBookMember_documentation_en-US" xlink:label="lab_efc_MeasurementInputEquityPricetoBookMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, Equity Price-to-Book [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MeasurementInputEquityPricetoBookMember" xlink:href="efc-20201231.xsd#efc_MeasurementInputEquityPricetoBookMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MeasurementInputEquityPricetoBookMember" xlink:to="lab_efc_MeasurementInputEquityPricetoBookMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InterestRateContractMember_9ac68a6d-c6a2-48b9-879e-520922b16f06_terseLabel_en-US" xlink:label="lab_us-gaap_InterestRateContractMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Rate Risk [Member]</link:label>
    <link:label id="lab_us-gaap_InterestRateContractMember_label_en-US" xlink:label="lab_us-gaap_InterestRateContractMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Rate Contract [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InterestRateContractMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InterestRateContractMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InterestRateContractMember" xlink:to="lab_us-gaap_InterestRateContractMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_UnrealizedGainLossonFinancialDerivativescurrencyhedges_3fb0d499-ee38-48a5-bea5-4f8833486271_terseLabel_en-US" xlink:label="lab_efc_UnrealizedGainLossonFinancialDerivativescurrencyhedges" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Derivatives-currency hedges</link:label>
    <link:label id="lab_efc_UnrealizedGainLossonFinancialDerivativescurrencyhedges_label_en-US" xlink:label="lab_efc_UnrealizedGainLossonFinancialDerivativescurrencyhedges" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized Gain (Loss) on Financial Derivatives, currency hedges</link:label>
    <link:label id="lab_efc_UnrealizedGainLossonFinancialDerivativescurrencyhedges_documentation_en-US" xlink:label="lab_efc_UnrealizedGainLossonFinancialDerivativescurrencyhedges" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized Gain (Loss) on Financial Derivatives, currency hedges</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_UnrealizedGainLossonFinancialDerivativescurrencyhedges" xlink:href="efc-20201231.xsd#efc_UnrealizedGainLossonFinancialDerivativescurrencyhedges"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_UnrealizedGainLossonFinancialDerivativescurrencyhedges" xlink:to="lab_efc_UnrealizedGainLossonFinancialDerivativescurrencyhedges" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RelatedPartyTransactionsAbstract_639dcdc7-5547-4d3f-9590-ec2c77448752_terseLabel_en-US" xlink:label="lab_us-gaap_RelatedPartyTransactionsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party Transactions [Abstract]</link:label>
    <link:label id="lab_us-gaap_RelatedPartyTransactionsAbstract_label_en-US" xlink:label="lab_us-gaap_RelatedPartyTransactionsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party Transactions [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RelatedPartyTransactionsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RelatedPartyTransactionsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RelatedPartyTransactionsAbstract" xlink:to="lab_us-gaap_RelatedPartyTransactionsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DeferredTaxAssetsValuationAllowance_844b6ca8-51c8-45cb-9f9a-3a7052a9759a_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_DeferredTaxAssetsValuationAllowance" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Tax Assets, Valuation Allowance</link:label>
    <link:label id="lab_us-gaap_DeferredTaxAssetsValuationAllowance_label_en-US" xlink:label="lab_us-gaap_DeferredTaxAssetsValuationAllowance" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Tax Assets, Valuation Allowance</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DeferredTaxAssetsValuationAllowance" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DeferredTaxAssetsValuationAllowance"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DeferredTaxAssetsValuationAllowance" xlink:to="lab_us-gaap_DeferredTaxAssetsValuationAllowance" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfCapitalizationEquityTable_a7217757-bc1e-4ad1-8e68-25ef3bc5ea20_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfCapitalizationEquityTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Capitalization, Equity [Table]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfCapitalizationEquityTable_label_en-US" xlink:label="lab_us-gaap_ScheduleOfCapitalizationEquityTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Capitalization, Equity [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfCapitalizationEquityTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfCapitalizationEquityTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfCapitalizationEquityTable" xlink:to="lab_us-gaap_ScheduleOfCapitalizationEquityTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Consumerloanspurchasedthroughaffiliate_0977d444-d25e-401f-9843-6f7661a794a9_terseLabel_en-US" xlink:label="lab_efc_Consumerloanspurchasedthroughaffiliate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consumer loans purchased through affiliate</link:label>
    <link:label id="lab_efc_Consumerloanspurchasedthroughaffiliate_label_en-US" xlink:label="lab_efc_Consumerloanspurchasedthroughaffiliate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consumer loans purchased through affiliate</link:label>
    <link:label id="lab_efc_Consumerloanspurchasedthroughaffiliate_documentation_en-US" xlink:label="lab_efc_Consumerloanspurchasedthroughaffiliate" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Aggregate principal balance of consumer loans purchased through an affiliate of the Company under purchase agreements.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Consumerloanspurchasedthroughaffiliate" xlink:href="efc-20201231.xsd#efc_Consumerloanspurchasedthroughaffiliate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Consumerloanspurchasedthroughaffiliate" xlink:to="lab_efc_Consumerloanspurchasedthroughaffiliate" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NoninterestExpenseDirectorsFees_5627b68f-3bcc-488a-a4f0-f371285ad808_terseLabel_en-US" xlink:label="lab_us-gaap_NoninterestExpenseDirectorsFees" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noninterest Expense Directors Fees</link:label>
    <link:label id="lab_us-gaap_NoninterestExpenseDirectorsFees_label_en-US" xlink:label="lab_us-gaap_NoninterestExpenseDirectorsFees" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noninterest Expense Directors Fees</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NoninterestExpenseDirectorsFees" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NoninterestExpenseDirectorsFees"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NoninterestExpenseDirectorsFees" xlink:to="lab_us-gaap_NoninterestExpenseDirectorsFees" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Estimatedremainingcontingentpurchaseobligations_3526d837-01c5-437d-9501-f260eb2687b5_terseLabel_en-US" xlink:label="lab_efc_Estimatedremainingcontingentpurchaseobligations" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Estimated remaining contingent purchase obligations</link:label>
    <link:label id="lab_efc_Estimatedremainingcontingentpurchaseobligations_label_en-US" xlink:label="lab_efc_Estimatedremainingcontingentpurchaseobligations" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Estimated remaining contingent purchase obligations</link:label>
    <link:label id="lab_efc_Estimatedremainingcontingentpurchaseobligations_documentation_en-US" xlink:label="lab_efc_Estimatedremainingcontingentpurchaseobligations" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Estimated remaining contingent purchase obligations of the Company under agreement with affiliate.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Estimatedremainingcontingentpurchaseobligations" xlink:href="efc-20201231.xsd#efc_Estimatedremainingcontingentpurchaseobligations"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Estimatedremainingcontingentpurchaseobligations" xlink:to="lab_efc_Estimatedremainingcontingentpurchaseobligations" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MarketBasedTotalReturn_f5777535-0252-41f2-b99b-b9ab98fa6692_terseLabel_en-US" xlink:label="lab_efc_MarketBasedTotalReturn" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Market Based Total Return</link:label>
    <link:label id="lab_efc_MarketBasedTotalReturn_label_en-US" xlink:label="lab_efc_MarketBasedTotalReturn" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Market Based Total Return</link:label>
    <link:label id="lab_efc_MarketBasedTotalReturn_documentation_en-US" xlink:label="lab_efc_MarketBasedTotalReturn" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total return for the period based upon the closing prices as reported by the New York Stock Exchange. Calculation of market based total return assumes the reinvestment of dividends at the closing price as reported by the New York Stock Exchange as of the ex-date.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MarketBasedTotalReturn" xlink:href="efc-20201231.xsd#efc_MarketBasedTotalReturn"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MarketBasedTotalReturn" xlink:to="lab_efc_MarketBasedTotalReturn" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_VariousIssuerMember_26cbce24-d525-49ac-aa11-7fb7b6ae5a93_terseLabel_en-US" xlink:label="lab_efc_VariousIssuerMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Various Issuer [Member]</link:label>
    <link:label id="lab_efc_VariousIssuerMember_label_en-US" xlink:label="lab_efc_VariousIssuerMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Various Issuer [Member]</link:label>
    <link:label id="lab_efc_VariousIssuerMember_documentation_en-US" xlink:label="lab_efc_VariousIssuerMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Various Issuer [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_VariousIssuerMember" xlink:href="efc-20201231.xsd#efc_VariousIssuerMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_VariousIssuerMember" xlink:to="lab_efc_VariousIssuerMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NoninterestIncome_91f8515e-54d4-4e20-9f9b-636ef5725416_totalLabel_en-US" xlink:label="lab_us-gaap_NoninterestIncome" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total other income (loss)</link:label>
    <link:label id="lab_us-gaap_NoninterestIncome_2b10ef2f-de9b-4c19-b098-2cea75d270fd_terseLabel_en-US" xlink:label="lab_us-gaap_NoninterestIncome" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noninterest Income</link:label>
    <link:label id="lab_us-gaap_NoninterestIncome_label_en-US" xlink:label="lab_us-gaap_NoninterestIncome" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noninterest Income</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NoninterestIncome" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NoninterestIncome"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NoninterestIncome" xlink:to="lab_us-gaap_NoninterestIncome" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ForeignCurrencyTransactionsAndTranslationsPolicyTextBlock_ce802c31-882b-4e8c-b392-526add6b4f7f_terseLabel_en-US" xlink:label="lab_us-gaap_ForeignCurrencyTransactionsAndTranslationsPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Foreign Currency</link:label>
    <link:label id="lab_us-gaap_ForeignCurrencyTransactionsAndTranslationsPolicyTextBlock_label_en-US" xlink:label="lab_us-gaap_ForeignCurrencyTransactionsAndTranslationsPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Foreign Currency Transactions and Translations Policy [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ForeignCurrencyTransactionsAndTranslationsPolicyTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ForeignCurrencyTransactionsAndTranslationsPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ForeignCurrencyTransactionsAndTranslationsPolicyTextBlock" xlink:to="lab_us-gaap_ForeignCurrencyTransactionsAndTranslationsPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PerformingCommercialMortgageLoansMember_eda9be72-9e8d-44d6-b5d0-b10e8ad1d8e2_terseLabel_en-US" xlink:label="lab_efc_PerformingCommercialMortgageLoansMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Performing commercial mortgage loans</link:label>
    <link:label id="lab_efc_PerformingCommercialMortgageLoansMember_label_en-US" xlink:label="lab_efc_PerformingCommercialMortgageLoansMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Performing Commercial Mortgage Loans [Member]</link:label>
    <link:label id="lab_efc_PerformingCommercialMortgageLoansMember_documentation_en-US" xlink:label="lab_efc_PerformingCommercialMortgageLoansMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commercial mortgage loans where the principal and/or interest payments are current.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PerformingCommercialMortgageLoansMember" xlink:href="efc-20201231.xsd#efc_PerformingCommercialMortgageLoansMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PerformingCommercialMortgageLoansMember" xlink:to="lab_efc_PerformingCommercialMortgageLoansMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MortgageLoansOnRealEstateAmortizationOfPremium_78ae387b-410d-47d7-9f7b-610c1fdb1359_terseLabel_en-US" xlink:label="lab_us-gaap_MortgageLoansOnRealEstateAmortizationOfPremium" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Amortization of Premium</link:label>
    <link:label id="lab_us-gaap_MortgageLoansOnRealEstateAmortizationOfPremium_label_en-US" xlink:label="lab_us-gaap_MortgageLoansOnRealEstateAmortizationOfPremium" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Amortization of Premium</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MortgageLoansOnRealEstateAmortizationOfPremium" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MortgageLoansOnRealEstateAmortizationOfPremium"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MortgageLoansOnRealEstateAmortizationOfPremium" xlink:to="lab_us-gaap_MortgageLoansOnRealEstateAmortizationOfPremium" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2025Member_60f490f6-7e2a-4076-b8df-dec714394e36_terseLabel_en-US" xlink:label="lab_efc_A2025Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2025 [Member]</link:label>
    <link:label id="lab_efc_A2025Member_label_en-US" xlink:label="lab_efc_A2025Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2025 [Member]</link:label>
    <link:label id="lab_efc_A2025Member_documentation_en-US" xlink:label="lab_efc_A2025Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2025 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2025Member" xlink:href="efc-20201231.xsd#efc_A2025Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2025Member" xlink:to="lab_efc_A2025Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentOwnedBalancePrincipalAmount_d8395920-f3d2-4c4b-b24d-914fcf3afd27_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentOwnedBalancePrincipalAmount" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unpaid Principal Balance</link:label>
    <link:label id="lab_us-gaap_InvestmentOwnedBalancePrincipalAmount_3319704f-1198-4750-aa10-6fd5e2c086ff_verboseLabel_en-US" xlink:label="lab_us-gaap_InvestmentOwnedBalancePrincipalAmount" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Current Principal/Notional Amount</link:label>
    <link:label id="lab_us-gaap_InvestmentOwnedBalancePrincipalAmount_label_en-US" xlink:label="lab_us-gaap_InvestmentOwnedBalancePrincipalAmount" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Owned, Balance, Principal Amount</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentOwnedBalancePrincipalAmount" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentOwnedBalancePrincipalAmount"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentOwnedBalancePrincipalAmount" xlink:to="lab_us-gaap_InvestmentOwnedBalancePrincipalAmount" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncreaseDecreaseInInterestPayableNet_81beb87f-ead0-4a9d-8149-559a0992c92e_terseLabel_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInInterestPayableNet" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (decrease) in interest payable</link:label>
    <link:label id="lab_us-gaap_IncreaseDecreaseInInterestPayableNet_label_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInInterestPayableNet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (Decrease) in Interest Payable, Net</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncreaseDecreaseInInterestPayableNet" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncreaseDecreaseInInterestPayableNet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncreaseDecreaseInInterestPayableNet" xlink:to="lab_us-gaap_IncreaseDecreaseInInterestPayableNet" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansOnRealEstateOriginalLoanAmountAxis_923f942f-b5eb-43e1-a292-44df546dfcc9_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateOriginalLoanAmountAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Original Amount [Axis]</link:label>
    <link:label id="lab_srt_MortgageLoansOnRealEstateOriginalLoanAmountAxis_label_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateOriginalLoanAmountAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Original Amount [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansOnRealEstateOriginalLoanAmountAxis" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansOnRealEstateOriginalLoanAmountAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansOnRealEstateOriginalLoanAmountAxis" xlink:to="lab_srt_MortgageLoansOnRealEstateOriginalLoanAmountAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OPLTIPUnitMember_bef2cba4-702a-42a6-b847-fa878e7d596d_terseLabel_en-US" xlink:label="lab_efc_OPLTIPUnitMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">OP LTIP Unit [Member]</link:label>
    <link:label id="lab_efc_OPLTIPUnitMember_label_en-US" xlink:label="lab_efc_OPLTIPUnitMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">OP LTIP Unit [Member]</link:label>
    <link:label id="lab_efc_OPLTIPUnitMember_documentation_en-US" xlink:label="lab_efc_OPLTIPUnitMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">OP LTIP Unit [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OPLTIPUnitMember" xlink:href="efc-20201231.xsd#efc_OPLTIPUnitMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OPLTIPUnitMember" xlink:to="lab_efc_OPLTIPUnitMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ClassificationOfVariableInterestEntityDomain_67fb72be-6c72-43d1-8be2-77e53214d5a7_terseLabel_en-US" xlink:label="lab_us-gaap_ClassificationOfVariableInterestEntityDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Variable Interest Entity, Classification [Domain]</link:label>
    <link:label id="lab_us-gaap_ClassificationOfVariableInterestEntityDomain_label_en-US" xlink:label="lab_us-gaap_ClassificationOfVariableInterestEntityDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Variable Interest Entity, Classification [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ClassificationOfVariableInterestEntityDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ClassificationOfVariableInterestEntityDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ClassificationOfVariableInterestEntityDomain" xlink:to="lab_us-gaap_ClassificationOfVariableInterestEntityDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ManagerMember_d3335865-696d-40b4-8a52-54af3fc480fc_terseLabel_en-US" xlink:label="lab_efc_ManagerMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Manager [Member]</link:label>
    <link:label id="lab_efc_ManagerMember_label_en-US" xlink:label="lab_efc_ManagerMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Manager [Member]</link:label>
    <link:label id="lab_efc_ManagerMember_documentation_en-US" xlink:label="lab_efc_ManagerMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Manager [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ManagerMember" xlink:href="efc-20201231.xsd#efc_ManagerMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ManagerMember" xlink:to="lab_efc_ManagerMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonExchangeTradedPreferredEquityInvestmentinloanoriginationentityMember_546cc53d-acd3-476d-9ba0-16f349151482_terseLabel_en-US" xlink:label="lab_efc_NonExchangeTradedPreferredEquityInvestmentinloanoriginationentityMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-exchange traded preferred equity investment in loan origination entity</link:label>
    <link:label id="lab_efc_NonExchangeTradedPreferredEquityInvestmentinloanoriginationentityMember_label_en-US" xlink:label="lab_efc_NonExchangeTradedPreferredEquityInvestmentinloanoriginationentityMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Preferred Equity Investment in loan origination entity [Member]</link:label>
    <link:label id="lab_efc_NonExchangeTradedPreferredEquityInvestmentinloanoriginationentityMember_documentation_en-US" xlink:label="lab_efc_NonExchangeTradedPreferredEquityInvestmentinloanoriginationentityMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Preferred Equity Investment in loan origination entity [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonExchangeTradedPreferredEquityInvestmentinloanoriginationentityMember" xlink:href="efc-20201231.xsd#efc_NonExchangeTradedPreferredEquityInvestmentinloanoriginationentityMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonExchangeTradedPreferredEquityInvestmentinloanoriginationentityMember" xlink:to="lab_efc_NonExchangeTradedPreferredEquityInvestmentinloanoriginationentityMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleOfPercentageOfTotalDepositsWithDealersHeldAsCollateralTableTextBlock_e6f776e8-a723-4950-a534-c43038041bce_terseLabel_en-US" xlink:label="lab_efc_ScheduleOfPercentageOfTotalDepositsWithDealersHeldAsCollateralTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">% Of Total Deposits With Dealers Held As Collateral</link:label>
    <link:label id="lab_efc_ScheduleOfPercentageOfTotalDepositsWithDealersHeldAsCollateralTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleOfPercentageOfTotalDepositsWithDealersHeldAsCollateralTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Percentage Of Total Deposits With Dealers Held As Collateral [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleOfPercentageOfTotalDepositsWithDealersHeldAsCollateralTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleOfPercentageOfTotalDepositsWithDealersHeldAsCollateralTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Tabular disclosure of the Company's percentage of total collateral on margin deposits held by counterparties above a specified concentration risk threshold.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleOfPercentageOfTotalDepositsWithDealersHeldAsCollateralTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleOfPercentageOfTotalDepositsWithDealersHeldAsCollateralTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleOfPercentageOfTotalDepositsWithDealersHeldAsCollateralTableTextBlock" xlink:to="lab_efc_ScheduleOfPercentageOfTotalDepositsWithDealersHeldAsCollateralTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Coparticipantsshareofloanssold_336e1b7f-fbb0-4479-a36c-e8ddcc951056_terseLabel_en-US" xlink:label="lab_efc_Coparticipantsshareofloanssold" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Co-participants share of loans sold</link:label>
    <link:label id="lab_efc_Coparticipantsshareofloanssold_label_en-US" xlink:label="lab_efc_Coparticipantsshareofloanssold" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Co-participants share of loans sold</link:label>
    <link:label id="lab_efc_Coparticipantsshareofloanssold_documentation_en-US" xlink:label="lab_efc_Coparticipantsshareofloanssold" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage of aggregate unpaid principal balance of consumer loans sold by the co-participant (affiliate of the Company) to Issuer in securitization transaction.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Coparticipantsshareofloanssold" xlink:href="efc-20201231.xsd#efc_Coparticipantsshareofloanssold"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Coparticipantsshareofloanssold" xlink:to="lab_efc_Coparticipantsshareofloanssold" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ConcentrationRiskLineItems_4f94b56b-18f0-4e7b-b685-f53366174590_terseLabel_en-US" xlink:label="lab_us-gaap_ConcentrationRiskLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk [Line Items]</link:label>
    <link:label id="lab_us-gaap_ConcentrationRiskLineItems_label_en-US" xlink:label="lab_us-gaap_ConcentrationRiskLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ConcentrationRiskLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ConcentrationRiskLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ConcentrationRiskLineItems" xlink:to="lab_us-gaap_ConcentrationRiskLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfRepurchaseAgreements_a3d3469d-471e-41c5-ad7a-06554d438cb4_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Repurchase Agreements [Table Text Block]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfRepurchaseAgreements_label_en-US" xlink:label="lab_us-gaap_ScheduleOfRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Repurchase Agreements [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfRepurchaseAgreements" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfRepurchaseAgreements"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfRepurchaseAgreements" xlink:to="lab_us-gaap_ScheduleOfRepurchaseAgreements" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NewAccountingPronouncementsPolicyPolicyTextBlock_3f0dce9d-863e-42e0-a97c-99a25093d722_terseLabel_en-US" xlink:label="lab_us-gaap_NewAccountingPronouncementsPolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Recent Accounting Pronouncements</link:label>
    <link:label id="lab_us-gaap_NewAccountingPronouncementsPolicyPolicyTextBlock_label_en-US" xlink:label="lab_us-gaap_NewAccountingPronouncementsPolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">New Accounting Pronouncements, Policy [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NewAccountingPronouncementsPolicyPolicyTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NewAccountingPronouncementsPolicyPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NewAccountingPronouncementsPolicyPolicyTextBlock" xlink:to="lab_us-gaap_NewAccountingPronouncementsPolicyPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RoyalBankofCanadaMember_4643c2a3-ea0c-486a-b076-a6de4e4b1b8b_terseLabel_en-US" xlink:label="lab_efc_RoyalBankofCanadaMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Royal Bank of Canada [Member]</link:label>
    <link:label id="lab_efc_RoyalBankofCanadaMember_label_en-US" xlink:label="lab_efc_RoyalBankofCanadaMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Royal Bank of Canada [Member]</link:label>
    <link:label id="lab_efc_RoyalBankofCanadaMember_documentation_en-US" xlink:label="lab_efc_RoyalBankofCanadaMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Royal Bank of Canada [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RoyalBankofCanadaMember" xlink:href="efc-20201231.xsd#efc_RoyalBankofCanadaMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RoyalBankofCanadaMember" xlink:to="lab_efc_RoyalBankofCanadaMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PaymentsForProceedsFromInvestments_156f7ad6-5da8-4944-9bb3-7c0d15db0af9_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_PaymentsForProceedsFromInvestments" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from disposition of investments</link:label>
    <link:label id="lab_us-gaap_PaymentsForProceedsFromInvestments_label_en-US" xlink:label="lab_us-gaap_PaymentsForProceedsFromInvestments" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments for (Proceeds from) Investments</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PaymentsForProceedsFromInvestments" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsForProceedsFromInvestments"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PaymentsForProceedsFromInvestments" xlink:to="lab_us-gaap_PaymentsForProceedsFromInvestments" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritizedResidentialMortgageLoansandREOMember_e3cbd8d8-0d0d-4b43-ad32-a9f3381bec6a_terseLabel_en-US" xlink:label="lab_efc_SecuritizedResidentialMortgageLoansandREOMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitized residential mortgage loans</link:label>
    <link:label id="lab_efc_SecuritizedResidentialMortgageLoansandREOMember_label_en-US" xlink:label="lab_efc_SecuritizedResidentialMortgageLoansandREOMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitized Residential Mortgage Loans and REO [Member]</link:label>
    <link:label id="lab_efc_SecuritizedResidentialMortgageLoansandREOMember_documentation_en-US" xlink:label="lab_efc_SecuritizedResidentialMortgageLoansandREOMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Residential mortgage loans and REO held in consolidated securitiztion trusts. [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritizedResidentialMortgageLoansandREOMember" xlink:href="efc-20201231.xsd#efc_SecuritizedResidentialMortgageLoansandREOMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritizedResidentialMortgageLoansandREOMember" xlink:to="lab_efc_SecuritizedResidentialMortgageLoansandREOMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancingReceivableCreditQualityIndicatorsTableTextBlock_f7e0706c-3433-443b-99bb-df303ffb73e6_terseLabel_en-US" xlink:label="lab_us-gaap_FinancingReceivableCreditQualityIndicatorsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable Credit Quality Indicators [Table Text Block]</link:label>
    <link:label id="lab_us-gaap_FinancingReceivableCreditQualityIndicatorsTableTextBlock_label_en-US" xlink:label="lab_us-gaap_FinancingReceivableCreditQualityIndicatorsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable Credit Quality Indicators [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancingReceivableCreditQualityIndicatorsTableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancingReceivableCreditQualityIndicatorsTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancingReceivableCreditQualityIndicatorsTableTextBlock" xlink:to="lab_us-gaap_FinancingReceivableCreditQualityIndicatorsTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfInvestmentsLineItems_e0ed0313-448c-4d4a-82e7-238df22c4847_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfInvestmentsLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Investments [Line Items]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfInvestmentsLineItems_label_en-US" xlink:label="lab_us-gaap_ScheduleOfInvestmentsLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Investments [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfInvestmentsLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfInvestmentsLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfInvestmentsLineItems" xlink:to="lab_us-gaap_ScheduleOfInvestmentsLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityAddressPostalZipCode_bfe4bc7e-a858-4347-909a-d4ff023edf27_terseLabel_en-US" xlink:label="lab_dei_EntityAddressPostalZipCode" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Address, Postal Zip Code</link:label>
    <link:label id="lab_dei_EntityAddressPostalZipCode_label_en-US" xlink:label="lab_dei_EntityAddressPostalZipCode" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Address, Postal Zip Code</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityAddressPostalZipCode" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityAddressPostalZipCode"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityAddressPostalZipCode" xlink:to="lab_dei_EntityAddressPostalZipCode" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CorporateDebtMember_9283ba83-eebd-4db7-9642-b74dfe4437f3_terseLabel_en-US" xlink:label="lab_efc_CorporateDebtMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate debt securities</link:label>
    <link:label id="lab_efc_CorporateDebtMember_label_en-US" xlink:label="lab_efc_CorporateDebtMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate Debt [Member]</link:label>
    <link:label id="lab_efc_CorporateDebtMember_documentation_en-US" xlink:label="lab_efc_CorporateDebtMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate debt instruments can include bank debt, direct debit investment in a corporation or private entity, bonds issued by corporations.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CorporateDebtMember" xlink:href="efc-20201231.xsd#efc_CorporateDebtMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CorporateDebtMember" xlink:to="lab_efc_CorporateDebtMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProceedsFromSecuritiesPurchasedUnderAgreementsToResell_2856d29a-630f-4511-94e8-202d9f50160d_terseLabel_en-US" xlink:label="lab_us-gaap_ProceedsFromSecuritiesPurchasedUnderAgreementsToResell" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from reverse repurchase agreements</link:label>
    <link:label id="lab_us-gaap_ProceedsFromSecuritiesPurchasedUnderAgreementsToResell_label_en-US" xlink:label="lab_us-gaap_ProceedsFromSecuritiesPurchasedUnderAgreementsToResell" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from Securities Purchased under Agreements to Resell</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProceedsFromSecuritiesPurchasedUnderAgreementsToResell" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromSecuritiesPurchasedUnderAgreementsToResell"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProceedsFromSecuritiesPurchasedUnderAgreementsToResell" xlink:to="lab_us-gaap_ProceedsFromSecuritiesPurchasedUnderAgreementsToResell" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PurchaseCommitmentRemainingMinimumAmountCommitted_5a32f565-4c53-464a-8935-f91f7e27a4f9_terseLabel_en-US" xlink:label="lab_us-gaap_PurchaseCommitmentRemainingMinimumAmountCommitted" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchase Commitment, Remaining Minimum Amount Committed</link:label>
    <link:label id="lab_us-gaap_PurchaseCommitmentRemainingMinimumAmountCommitted_label_en-US" xlink:label="lab_us-gaap_PurchaseCommitmentRemainingMinimumAmountCommitted" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchase Commitment, Remaining Minimum Amount Committed</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PurchaseCommitmentRemainingMinimumAmountCommitted" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PurchaseCommitmentRemainingMinimumAmountCommitted"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PurchaseCommitmentRemainingMinimumAmountCommitted" xlink:to="lab_us-gaap_PurchaseCommitmentRemainingMinimumAmountCommitted" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CreditDerivativesContractTypeDomain_00e08e83-6328-4a81-8225-f140253bb0bb_terseLabel_en-US" xlink:label="lab_us-gaap_CreditDerivativesContractTypeDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Derivatives Contract Type [Domain]</link:label>
    <link:label id="lab_us-gaap_CreditDerivativesContractTypeDomain_label_en-US" xlink:label="lab_us-gaap_CreditDerivativesContractTypeDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Derivatives Contract Type [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CreditDerivativesContractTypeDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CreditDerivativesContractTypeDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CreditDerivativesContractTypeDomain" xlink:to="lab_us-gaap_CreditDerivativesContractTypeDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AvailableForSaleSecuritiesAmortizedCost_6cb2578d-c111-4d0b-841e-670d9cb5a6d8_terseLabel_en-US" xlink:label="lab_us-gaap_AvailableForSaleSecuritiesAmortizedCost" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amortized Cost</link:label>
    <link:label id="lab_us-gaap_AvailableForSaleSecuritiesAmortizedCost_label_en-US" xlink:label="lab_us-gaap_AvailableForSaleSecuritiesAmortizedCost" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Available-for-sale Securities, Amortized Cost Basis</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AvailableForSaleSecuritiesAmortizedCost" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AvailableForSaleSecuritiesAmortizedCost"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AvailableForSaleSecuritiesAmortizedCost" xlink:to="lab_us-gaap_AvailableForSaleSecuritiesAmortizedCost" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OptionsMember_1e0352f6-72d7-43e3-87e5-39533deb349e_terseLabel_en-US" xlink:label="lab_efc_OptionsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Options</link:label>
    <link:label id="lab_efc_OptionsMember_label_en-US" xlink:label="lab_efc_OptionsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Options [Member]</link:label>
    <link:label id="lab_efc_OptionsMember_documentation_en-US" xlink:label="lab_efc_OptionsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Contracts conveying rights, but not obligations, to buy or sell a specific commodity, or financial or equity instrument, at a specified price during a specified period (an American option) or at a specified date (a European option)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OptionsMember" xlink:href="efc-20201231.xsd#efc_OptionsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OptionsMember" xlink:to="lab_efc_OptionsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentSoldNotYetPurchasedBalanceShares_1b0e1d67-24e6-46c2-97e0-2a76449745a8_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_InvestmentSoldNotYetPurchasedBalanceShares" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Sold, Not yet Purchased, Balance, Shares</link:label>
    <link:label id="lab_us-gaap_InvestmentSoldNotYetPurchasedBalanceShares_label_en-US" xlink:label="lab_us-gaap_InvestmentSoldNotYetPurchasedBalanceShares" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Sold, Not yet Purchased, Balance, Shares</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentSoldNotYetPurchasedBalanceShares" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentSoldNotYetPurchasedBalanceShares"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentSoldNotYetPurchasedBalanceShares" xlink:to="lab_us-gaap_InvestmentSoldNotYetPurchasedBalanceShares" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentsBySecondaryCategorizationDomain_7f9930d2-b6e7-422b-8b70-07b7ce99f56a_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentsBySecondaryCategorizationDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments by Secondary Categorization [Domain]</link:label>
    <link:label id="lab_us-gaap_InvestmentsBySecondaryCategorizationDomain_label_en-US" xlink:label="lab_us-gaap_InvestmentsBySecondaryCategorizationDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments by Secondary Categorization [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentsBySecondaryCategorizationDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentsBySecondaryCategorizationDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentsBySecondaryCategorizationDomain" xlink:to="lab_us-gaap_InvestmentsBySecondaryCategorizationDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CreditDefaultSwapMember_3afb0c59-710f-4ced-b8a3-b03276279d9e_terseLabel_en-US" xlink:label="lab_us-gaap_CreditDefaultSwapMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit default swaps</link:label>
    <link:label id="lab_us-gaap_CreditDefaultSwapMember_label_en-US" xlink:label="lab_us-gaap_CreditDefaultSwapMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swap [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CreditDefaultSwapMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CreditDefaultSwapMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CreditDefaultSwapMember" xlink:to="lab_us-gaap_CreditDefaultSwapMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquityMethodInvestmentsPolicy_b2722d67-7b5b-4859-a022-2fe116a9b11a_terseLabel_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentsPolicy" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments in Unconsolidated Entities</link:label>
    <link:label id="lab_us-gaap_EquityMethodInvestmentsPolicy_label_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentsPolicy" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investments [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquityMethodInvestmentsPolicy" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquityMethodInvestmentsPolicy"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquityMethodInvestmentsPolicy" xlink:to="lab_us-gaap_EquityMethodInvestmentsPolicy" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfAvailableForSaleSecuritiesLineItems_57f73efb-1507-4475-98c7-81dc5ce2c6d0_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfAvailableForSaleSecuritiesLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities, Available-for-sale [Line Items]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfAvailableForSaleSecuritiesLineItems_label_en-US" xlink:label="lab_us-gaap_ScheduleOfAvailableForSaleSecuritiesLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfAvailableForSaleSecuritiesLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfAvailableForSaleSecuritiesLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfAvailableForSaleSecuritiesLineItems" xlink:to="lab_us-gaap_ScheduleOfAvailableForSaleSecuritiesLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_Assets_c69c5b96-715b-415c-82a7-f9eddf517ff5_totalLabel_en-US" xlink:label="lab_us-gaap_Assets" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Assets</link:label>
    <link:label id="lab_us-gaap_Assets_label_en-US" xlink:label="lab_us-gaap_Assets" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_Assets" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_Assets"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_Assets" xlink:to="lab_us-gaap_Assets" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CreditDefaultSwapsAssetBackedSecuritiesMember_33cb3f4e-6ed7-4e19-a946-0d3f446c5cdc_terseLabel_en-US" xlink:label="lab_efc_CreditDefaultSwapsAssetBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit default swaps on asset-backed securities</link:label>
    <link:label id="lab_efc_CreditDefaultSwapsAssetBackedSecuritiesMember_label_en-US" xlink:label="lab_efc_CreditDefaultSwapsAssetBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swaps, Asset-Backed Securities [Member]</link:label>
    <link:label id="lab_efc_CreditDefaultSwapsAssetBackedSecuritiesMember_documentation_en-US" xlink:label="lab_efc_CreditDefaultSwapsAssetBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swaps, Asset-Backed Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CreditDefaultSwapsAssetBackedSecuritiesMember" xlink:href="efc-20201231.xsd#efc_CreditDefaultSwapsAssetBackedSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CreditDefaultSwapsAssetBackedSecuritiesMember" xlink:to="lab_efc_CreditDefaultSwapsAssetBackedSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PreferredStockLiquidationPreferenceValue_bac74813-6263-4fe4-8864-00756e539365_terseLabel_en-US" xlink:label="lab_us-gaap_PreferredStockLiquidationPreferenceValue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Liquidation Preference, Value</link:label>
    <link:label id="lab_us-gaap_PreferredStockLiquidationPreferenceValue_label_en-US" xlink:label="lab_us-gaap_PreferredStockLiquidationPreferenceValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Liquidation Preference, Value</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PreferredStockLiquidationPreferenceValue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PreferredStockLiquidationPreferenceValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PreferredStockLiquidationPreferenceValue" xlink:to="lab_us-gaap_PreferredStockLiquidationPreferenceValue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationTotalotherincomeloss_476b1916-1de8-4ff3-b016-fc682ed69ee4_terseLabel_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationTotalotherincomeloss" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Total other income (loss)</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationTotalotherincomeloss_label_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationTotalotherincomeloss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Total other income (loss)</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationTotalotherincomeloss_documentation_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationTotalotherincomeloss" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount of total other income (loss) (including realized and unrealized gains (losses) on investments) reported by equity method investee.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationTotalotherincomeloss" xlink:href="efc-20201231.xsd#efc_EquityMethodInvestmentSummarizedFinancialInformationTotalotherincomeloss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationTotalotherincomeloss" xlink:to="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationTotalotherincomeloss" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityShellCompany_f0d6f477-c028-43f7-8e5b-9459d376a85c_terseLabel_en-US" xlink:label="lab_dei_EntityShellCompany" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Shell Company</link:label>
    <link:label id="lab_dei_EntityShellCompany_label_en-US" xlink:label="lab_dei_EntityShellCompany" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Shell Company</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityShellCompany" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityShellCompany"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityShellCompany" xlink:to="lab_dei_EntityShellCompany" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CreditDefaultSwapsOnCorporateBondIndicesMember_cd306903-37cd-444e-a8ee-48851f2d317f_terseLabel_en-US" xlink:label="lab_efc_CreditDefaultSwapsOnCorporateBondIndicesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit default swaps on corporate bond indices</link:label>
    <link:label id="lab_efc_CreditDefaultSwapsOnCorporateBondIndicesMember_label_en-US" xlink:label="lab_efc_CreditDefaultSwapsOnCorporateBondIndicesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swaps On Corporate Bond Indices [Member]</link:label>
    <link:label id="lab_efc_CreditDefaultSwapsOnCorporateBondIndicesMember_documentation_en-US" xlink:label="lab_efc_CreditDefaultSwapsOnCorporateBondIndicesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swaps On Corporate Bond Indices [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CreditDefaultSwapsOnCorporateBondIndicesMember" xlink:href="efc-20201231.xsd#efc_CreditDefaultSwapsOnCorporateBondIndicesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CreditDefaultSwapsOnCorporateBondIndicesMember" xlink:to="lab_efc_CreditDefaultSwapsOnCorporateBondIndicesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SubsequentEventMember_f29f31cc-67e6-44a0-b64b-08c78fce5ce4_terseLabel_en-US" xlink:label="lab_us-gaap_SubsequentEventMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsequent Event [Member]</link:label>
    <link:label id="lab_us-gaap_SubsequentEventMember_label_en-US" xlink:label="lab_us-gaap_SubsequentEventMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsequent Event [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SubsequentEventMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SubsequentEventMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SubsequentEventMember" xlink:to="lab_us-gaap_SubsequentEventMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DisclosureOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTextBlock_de7c0da4-5d41-41c0-a56d-e2c14a3882e7_verboseLabel_en-US" xlink:label="lab_us-gaap_DisclosureOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTextBlock" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Roll-Forward of Company's LTIP Units Outstanding</link:label>
    <link:label id="lab_us-gaap_DisclosureOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTextBlock_label_en-US" xlink:label="lab_us-gaap_DisclosureOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disclosure of Share-based Compensation Arrangements by Share-based Payment Award [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DisclosureOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DisclosureOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DisclosureOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTextBlock" xlink:to="lab_us-gaap_DisclosureOfShareBasedCompensationArrangementsByShareBasedPaymentAwardTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritiesclassifiedbyestimatedremaininglifeTableTextBlock_abf7f798-4bf6-4ae5-9bdf-f207073dc125_terseLabel_en-US" xlink:label="lab_efc_SecuritiesclassifiedbyestimatedremaininglifeTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities by Weighted Average Life</link:label>
    <link:label id="lab_efc_SecuritiesclassifiedbyestimatedremaininglifeTableTextBlock_label_en-US" xlink:label="lab_efc_SecuritiesclassifiedbyestimatedremaininglifeTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities classified by estimated remaining life [Table Text Block]</link:label>
    <link:label id="lab_efc_SecuritiesclassifiedbyestimatedremaininglifeTableTextBlock_documentation_en-US" xlink:label="lab_efc_SecuritiesclassifiedbyestimatedremaininglifeTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Tabular disclosure for securities by range of estimated weighted average remaining life.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritiesclassifiedbyestimatedremaininglifeTableTextBlock" xlink:href="efc-20201231.xsd#efc_SecuritiesclassifiedbyestimatedremaininglifeTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritiesclassifiedbyestimatedremaininglifeTableTextBlock" xlink:to="lab_efc_SecuritiesclassifiedbyestimatedremaininglifeTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotOffsetPolicyElectionDeduction_91292614-7007-4f53-91f4-693549cb0950_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotOffsetPolicyElectionDeduction" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instruments Available for Offset</link:label>
    <link:label id="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotOffsetPolicyElectionDeduction_label_en-US" xlink:label="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotOffsetPolicyElectionDeduction" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Purchased under Agreements to Resell, Not Offset, Policy Election Deduction</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotOffsetPolicyElectionDeduction" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotOffsetPolicyElectionDeduction"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotOffsetPolicyElectionDeduction" xlink:to="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotOffsetPolicyElectionDeduction" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year3Point00PercentMember_ce2fe8b6-c6a4-4ebc-9e92-dc7263699153_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year3Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 3 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year3Point00PercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year3Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 3 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year3Point00PercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year3Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 3 Point 00 Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociation30Year3Point00PercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociation30Year3Point00PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociation30Year3Point00PercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociation30Year3Point00PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_BarclaysCapitalIncCollateralizedbyParValue1200ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member_7bebd0ea-5fda-4196-a862-4b3d88dec370_terseLabel_en-US" xlink:label="lab_efc_BarclaysCapitalIncCollateralizedbyParValue1200ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Barclays Capital Inc Collateralized by Par Value $1,200 Exchange-Traded Corporate Debt, Coupon 9.88%, Maturity Date 2024 02 [Member]</link:label>
    <link:label id="lab_efc_BarclaysCapitalIncCollateralizedbyParValue1200ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member_label_en-US" xlink:label="lab_efc_BarclaysCapitalIncCollateralizedbyParValue1200ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Barclays Capital Inc Collateralized by Par Value $1,200 Exchange-Traded Corporate Debt, Coupon 9.88%, Maturity Date 2024 02 [Member]</link:label>
    <link:label id="lab_efc_BarclaysCapitalIncCollateralizedbyParValue1200ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member_documentation_en-US" xlink:label="lab_efc_BarclaysCapitalIncCollateralizedbyParValue1200ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Barclays Capital Inc Collateralized by Par Value $1,200 Exchange-Traded Corporate Debt, Coupon 9.88%, Maturity Date 2024 02 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_BarclaysCapitalIncCollateralizedbyParValue1200ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member" xlink:href="efc-20201231.xsd#efc_BarclaysCapitalIncCollateralizedbyParValue1200ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_BarclaysCapitalIncCollateralizedbyParValue1200ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member" xlink:to="lab_efc_BarclaysCapitalIncCollateralizedbyParValue1200ExchangeTradedCorporateDebtCoupon9.88MaturityDate202402Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FixedRateCommercialMortgageMember_59b9d478-2aa3-4f08-9b1b-ac9c27e4d60d_terseLabel_en-US" xlink:label="lab_efc_FixedRateCommercialMortgageMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed Rate commercial Mortgage</link:label>
    <link:label id="lab_efc_FixedRateCommercialMortgageMember_label_en-US" xlink:label="lab_efc_FixedRateCommercialMortgageMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed Rate commercial Mortgage [Member]</link:label>
    <link:label id="lab_efc_FixedRateCommercialMortgageMember_documentation_en-US" xlink:label="lab_efc_FixedRateCommercialMortgageMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed Rate commercial Mortgage</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FixedRateCommercialMortgageMember" xlink:href="efc-20201231.xsd#efc_FixedRateCommercialMortgageMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FixedRateCommercialMortgageMember" xlink:to="lab_efc_FixedRateCommercialMortgageMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritiesAndLoansUnrealizedGainsLosses_31e09bc2-e9f7-4da9-bbb9-168e3c8a7f57_terseLabel_en-US" xlink:label="lab_efc_SecuritiesAndLoansUnrealizedGainsLosses" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized gains (losses) on securities and loans, net</link:label>
    <link:label id="lab_efc_SecuritiesAndLoansUnrealizedGainsLosses_67d068ba-e1b7-4460-bdf8-f43fd4fe33ee_negatedTerseLabel_en-US" xlink:label="lab_efc_SecuritiesAndLoansUnrealizedGainsLosses" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized (gains) losses on securities and loans, net</link:label>
    <link:label id="lab_efc_SecuritiesAndLoansUnrealizedGainsLosses_label_en-US" xlink:label="lab_efc_SecuritiesAndLoansUnrealizedGainsLosses" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities And Loans, Unrealized Gains (Losses)</link:label>
    <link:label id="lab_efc_SecuritiesAndLoansUnrealizedGainsLosses_documentation_en-US" xlink:label="lab_efc_SecuritiesAndLoansUnrealizedGainsLosses" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities And Loans, Unrealized Gains (Losses)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritiesAndLoansUnrealizedGainsLosses" xlink:href="efc-20201231.xsd#efc_SecuritiesAndLoansUnrealizedGainsLosses"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritiesAndLoansUnrealizedGainsLosses" xlink:to="lab_efc_SecuritiesAndLoansUnrealizedGainsLosses" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_PA_4d784c18-1563-428f-addd-407807deed82_terseLabel_en-US" xlink:label="lab_stpr_PA" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">NEW JERSEY</link:label>
    <link:label id="lab_stpr_PA_label_en-US" xlink:label="lab_stpr_PA" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">PENNSYLVANIA</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_PA" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_PA"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_PA" xlink:to="lab_stpr_PA" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InterestPayableCurrentAndNoncurrent_47886503-91b8-4c3c-a12c-5293cff79fa4_terseLabel_en-US" xlink:label="lab_us-gaap_InterestPayableCurrentAndNoncurrent" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest payable</link:label>
    <link:label id="lab_us-gaap_InterestPayableCurrentAndNoncurrent_label_en-US" xlink:label="lab_us-gaap_InterestPayableCurrentAndNoncurrent" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Payable</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InterestPayableCurrentAndNoncurrent" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InterestPayableCurrentAndNoncurrent"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InterestPayableCurrentAndNoncurrent" xlink:to="lab_us-gaap_InterestPayableCurrentAndNoncurrent" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ComparabilityOfPriorYearFinancialData_7f658d23-a978-4354-b456-d901fbc4e89b_terseLabel_en-US" xlink:label="lab_us-gaap_ComparabilityOfPriorYearFinancialData" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reclassification and Presentation</link:label>
    <link:label id="lab_us-gaap_ComparabilityOfPriorYearFinancialData_label_en-US" xlink:label="lab_us-gaap_ComparabilityOfPriorYearFinancialData" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Comparability of Prior Year Financial Data, Policy [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ComparabilityOfPriorYearFinancialData" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ComparabilityOfPriorYearFinancialData"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ComparabilityOfPriorYearFinancialData" xlink:to="lab_us-gaap_ComparabilityOfPriorYearFinancialData" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesInvestmentMember_b4d4f161-58af-4d85-9cd6-b381764d7dde_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesInvestmentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Investment [Member]</link:label>
    <link:label id="lab_us-gaap_SecuritiesInvestmentMember_label_en-US" xlink:label="lab_us-gaap_SecuritiesInvestmentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Investment [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesInvestmentMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesInvestmentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesInvestmentMember" xlink:to="lab_us-gaap_SecuritiesInvestmentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ForeignGovernmentDebtSecuritiesMember_b1b82eca-0db8-4e84-b085-21afee23fe46_terseLabel_en-US" xlink:label="lab_us-gaap_ForeignGovernmentDebtSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Security, Government, Non-US [Member]</link:label>
    <link:label id="lab_us-gaap_ForeignGovernmentDebtSecuritiesMember_label_en-US" xlink:label="lab_us-gaap_ForeignGovernmentDebtSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Security, Government, Non-US [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ForeignGovernmentDebtSecuritiesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ForeignGovernmentDebtSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ForeignGovernmentDebtSecuritiesMember" xlink:to="lab_us-gaap_ForeignGovernmentDebtSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansBetween750000And999999Member_5c5c794f-b77a-4743-aa84-872dbd479832_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansBetween750000And999999Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $750,000 and $999,999</link:label>
    <link:label id="lab_srt_MortgageLoansBetween750000And999999Member_label_en-US" xlink:label="lab_srt_MortgageLoansBetween750000And999999Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $750,000 and $999,999 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansBetween750000And999999Member" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansBetween750000And999999Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansBetween750000And999999Member" xlink:to="lab_srt_MortgageLoansBetween750000And999999Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss_7e1cb699-119d-48fd-a4a4-87162245d2ab_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized Losses, Greater than 12 Months</link:label>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss_label_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Continuous Unrealized Loss Position, 12 Months or Longer, Accumulated Loss</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss" xlink:to="lab_us-gaap_DebtSecuritiesAvailableForSaleContinuousUnrealizedLossPosition12MonthsOrLongerAccumulatedLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Seniornotesissuedinsecuritizationtransaction_505f63c0-5f4c-4730-93cd-8227c87ca697_terseLabel_en-US" xlink:label="lab_efc_Seniornotesissuedinsecuritizationtransaction" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Senior notes issued in securitization transaction</link:label>
    <link:label id="lab_efc_Seniornotesissuedinsecuritizationtransaction_label_en-US" xlink:label="lab_efc_Seniornotesissuedinsecuritizationtransaction" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Senior notes issued in securitization transaction</link:label>
    <link:label id="lab_efc_Seniornotesissuedinsecuritizationtransaction_documentation_en-US" xlink:label="lab_efc_Seniornotesissuedinsecuritizationtransaction" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Original face amount of senior notes issued pursuant to the securitization.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Seniornotesissuedinsecuritizationtransaction" xlink:href="efc-20201231.xsd#efc_Seniornotesissuedinsecuritizationtransaction"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Seniornotesissuedinsecuritizationtransaction" xlink:to="lab_efc_Seniornotesissuedinsecuritizationtransaction" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MortgageBackedSecuritiesIssuedByPrivateEnterprisesMember_3a47ef4a-2c45-44b7-8423-e1a38242b8a8_terseLabel_en-US" xlink:label="lab_us-gaap_MortgageBackedSecuritiesIssuedByPrivateEnterprisesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Agency RMBS</link:label>
    <link:label id="lab_us-gaap_MortgageBackedSecuritiesIssuedByPrivateEnterprisesMember_label_en-US" xlink:label="lab_us-gaap_MortgageBackedSecuritiesIssuedByPrivateEnterprisesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage-backed Securities, Issued by Private Enterprises [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MortgageBackedSecuritiesIssuedByPrivateEnterprisesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MortgageBackedSecuritiesIssuedByPrivateEnterprisesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MortgageBackedSecuritiesIssuedByPrivateEnterprisesMember" xlink:to="lab_us-gaap_MortgageBackedSecuritiesIssuedByPrivateEnterprisesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProceedsFromCollectionOfLoansReceivable_f24e3b7b-1bab-47df-aa8f-df1fdd4a40fd_terseLabel_en-US" xlink:label="lab_us-gaap_ProceedsFromCollectionOfLoansReceivable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from principal payments of loans</link:label>
    <link:label id="lab_us-gaap_ProceedsFromCollectionOfLoansReceivable_label_en-US" xlink:label="lab_us-gaap_ProceedsFromCollectionOfLoansReceivable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from Collection of Loans Receivable</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProceedsFromCollectionOfLoansReceivable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromCollectionOfLoansReceivable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProceedsFromCollectionOfLoansReceivable" xlink:to="lab_us-gaap_ProceedsFromCollectionOfLoansReceivable" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_OR_8737eb45-14ea-4567-beb1-93805a63d85a_terseLabel_en-US" xlink:label="lab_stpr_OR" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">OREGON</link:label>
    <link:label id="lab_stpr_OR_label_en-US" xlink:label="lab_stpr_OR" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">OREGON</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_OR" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_OR"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_OR" xlink:to="lab_stpr_OR" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePercentMember_ba627dd7-7cff-4159-86c5-fd9b47475516_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Three Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Three Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Three Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearThreePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LongtermDebtTypeDomain_13366789-7a5d-43b8-b6d9-ab3421d5057d_terseLabel_en-US" xlink:label="lab_us-gaap_LongtermDebtTypeDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Type [Domain]</link:label>
    <link:label id="lab_us-gaap_LongtermDebtTypeDomain_label_en-US" xlink:label="lab_us-gaap_LongtermDebtTypeDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Type [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LongtermDebtTypeDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LongtermDebtTypeDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LongtermDebtTypeDomain" xlink:to="lab_us-gaap_LongtermDebtTypeDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritiespurchasedunderagreementstoresellAxis_f0de3194-67d2-4be5-95e2-bb85fafdc236_terseLabel_en-US" xlink:label="lab_efc_SecuritiespurchasedunderagreementstoresellAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities purchased under agreements to resell [Axis]</link:label>
    <link:label id="lab_efc_SecuritiespurchasedunderagreementstoresellAxis_label_en-US" xlink:label="lab_efc_SecuritiespurchasedunderagreementstoresellAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities purchased under agreements to resell [Axis]</link:label>
    <link:label id="lab_efc_SecuritiespurchasedunderagreementstoresellAxis_documentation_en-US" xlink:label="lab_efc_SecuritiespurchasedunderagreementstoresellAxis" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities purchased under agreements to resell [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritiespurchasedunderagreementstoresellAxis" xlink:href="efc-20201231.xsd#efc_SecuritiespurchasedunderagreementstoresellAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritiespurchasedunderagreementstoresellAxis" xlink:to="lab_efc_SecuritiespurchasedunderagreementstoresellAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProfessionalFees_74c55805-b14d-4a4a-82bd-11501a6b6079_terseLabel_en-US" xlink:label="lab_us-gaap_ProfessionalFees" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Professional fees</link:label>
    <link:label id="lab_us-gaap_ProfessionalFees_label_en-US" xlink:label="lab_us-gaap_ProfessionalFees" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Professional Fees</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProfessionalFees" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProfessionalFees"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProfessionalFees" xlink:to="lab_us-gaap_ProfessionalFees" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MeasurementinputprojectedcollateralscheduledamortizationMember_9c4ab410-51f2-4802-9582-0e31c2a6cff0_terseLabel_en-US" xlink:label="lab_efc_MeasurementinputprojectedcollateralscheduledamortizationMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement input, Projected Collateral Scheduled Amortization</link:label>
    <link:label id="lab_efc_MeasurementinputprojectedcollateralscheduledamortizationMember_label_en-US" xlink:label="lab_efc_MeasurementinputprojectedcollateralscheduledamortizationMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement input, projected collateral scheduled amortization [Member]</link:label>
    <link:label id="lab_efc_MeasurementinputprojectedcollateralscheduledamortizationMember_documentation_en-US" xlink:label="lab_efc_MeasurementinputprojectedcollateralscheduledamortizationMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Projected collateral repayment after projected prepayments, losses and severities, expressed as a percentage of collateral's current principal balance and used as an input to fair value.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MeasurementinputprojectedcollateralscheduledamortizationMember" xlink:href="efc-20201231.xsd#efc_MeasurementinputprojectedcollateralscheduledamortizationMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MeasurementinputprojectedcollateralscheduledamortizationMember" xlink:to="lab_efc_MeasurementinputprojectedcollateralscheduledamortizationMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CumulativePreferredStockMember_9c933bd5-d03c-49ff-a557-b612bde5e687_terseLabel_en-US" xlink:label="lab_us-gaap_CumulativePreferredStockMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cumulative Redeemable Preferred Stock</link:label>
    <link:label id="lab_us-gaap_CumulativePreferredStockMember_label_en-US" xlink:label="lab_us-gaap_CumulativePreferredStockMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cumulative Preferred Stock [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CumulativePreferredStockMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CumulativePreferredStockMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CumulativePreferredStockMember" xlink:to="lab_us-gaap_CumulativePreferredStockMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OperatingPartnershipUnitsIssued_b3f7ef12-d784-4333-aaf9-0cac78387836_terseLabel_en-US" xlink:label="lab_efc_OperatingPartnershipUnitsIssued" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership Units Issued</link:label>
    <link:label id="lab_efc_OperatingPartnershipUnitsIssued_label_en-US" xlink:label="lab_efc_OperatingPartnershipUnitsIssued" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership Units Issued</link:label>
    <link:label id="lab_efc_OperatingPartnershipUnitsIssued_documentation_en-US" xlink:label="lab_efc_OperatingPartnershipUnitsIssued" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership Units issued.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OperatingPartnershipUnitsIssued" xlink:href="efc-20201231.xsd#efc_OperatingPartnershipUnitsIssued"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OperatingPartnershipUnitsIssued" xlink:to="lab_efc_OperatingPartnershipUnitsIssued" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ConsolidationVariableInterestEntityPolicy_bcb4e98b-3855-469a-981d-10d1ceaec41c_terseLabel_en-US" xlink:label="lab_us-gaap_ConsolidationVariableInterestEntityPolicy" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Variable Interest Entities</link:label>
    <link:label id="lab_us-gaap_ConsolidationVariableInterestEntityPolicy_label_en-US" xlink:label="lab_us-gaap_ConsolidationVariableInterestEntityPolicy" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidation, Variable Interest Entity, Policy [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ConsolidationVariableInterestEntityPolicy" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ConsolidationVariableInterestEntityPolicy"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ConsolidationVariableInterestEntityPolicy" xlink:to="lab_us-gaap_ConsolidationVariableInterestEntityPolicy" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue463U.S.TreasuryNoteCoupon2.63MaturityDate202306Member_48fcf007-1561-4ea5-b5f2-a35813dda346_terseLabel_en-US" xlink:label="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue463U.S.TreasuryNoteCoupon2.63MaturityDate202306Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of America Securities Collateralized by Par Value $463 U.S. Treasury Note, Coupon 2.63%, Maturity Date 2023 06 [Member]</link:label>
    <link:label id="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue463U.S.TreasuryNoteCoupon2.63MaturityDate202306Member_label_en-US" xlink:label="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue463U.S.TreasuryNoteCoupon2.63MaturityDate202306Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of America Securities Collateralized by Par Value $463 U.S. Treasury Note, Coupon 2.63%, Maturity Date 2023 06 [Member]</link:label>
    <link:label id="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue463U.S.TreasuryNoteCoupon2.63MaturityDate202306Member_documentation_en-US" xlink:label="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue463U.S.TreasuryNoteCoupon2.63MaturityDate202306Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Bank of America Securities Collateralized by Par Value $463 U.S. Treasury Note, Coupon 2.63%, Maturity Date 2023 06 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_BankofAmericaSecuritiesCollateralizedbyParValue463U.S.TreasuryNoteCoupon2.63MaturityDate202306Member" xlink:href="efc-20201231.xsd#efc_BankofAmericaSecuritiesCollateralizedbyParValue463U.S.TreasuryNoteCoupon2.63MaturityDate202306Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_BankofAmericaSecuritiesCollateralizedbyParValue463U.S.TreasuryNoteCoupon2.63MaturityDate202306Member" xlink:to="lab_efc_BankofAmericaSecuritiesCollateralizedbyParValue463U.S.TreasuryNoteCoupon2.63MaturityDate202306Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationMember_1aa2fce6-051f-41e0-bfe0-07ff5b887693_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal home loan mortgage corporation [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ClassOfStockDomain_ddef767f-c0da-4bb4-9b25-95b5d8c43ff3_terseLabel_en-US" xlink:label="lab_us-gaap_ClassOfStockDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Class of Stock [Domain]</link:label>
    <link:label id="lab_us-gaap_ClassOfStockDomain_label_en-US" xlink:label="lab_us-gaap_ClassOfStockDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Class of Stock [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ClassOfStockDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ClassOfStockDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ClassOfStockDomain" xlink:to="lab_us-gaap_ClassOfStockDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DisposalGroupClassificationAxis_a5fd79c2-0637-4019-a7e3-846da0a441d6_terseLabel_en-US" xlink:label="lab_us-gaap_DisposalGroupClassificationAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disposal Group Classification [Axis]</link:label>
    <link:label id="lab_us-gaap_DisposalGroupClassificationAxis_label_en-US" xlink:label="lab_us-gaap_DisposalGroupClassificationAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disposal Group Classification [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DisposalGroupClassificationAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DisposalGroupClassificationAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DisposalGroupClassificationAxis" xlink:to="lab_us-gaap_DisposalGroupClassificationAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Assetssoldunderagreementstorepurchaseremainingdaystomaturity_100a9182-6637-47fd-a418-a93e43e99150_terseLabel_en-US" xlink:label="lab_efc_Assetssoldunderagreementstorepurchaseremainingdaystomaturity" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase agreements, remaining days to maturity</link:label>
    <link:label id="lab_efc_Assetssoldunderagreementstorepurchaseremainingdaystomaturity_label_en-US" xlink:label="lab_efc_Assetssoldunderagreementstorepurchaseremainingdaystomaturity" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets sold under agreements to repurchase, remaining days to maturity</link:label>
    <link:label id="lab_efc_Assetssoldunderagreementstorepurchaseremainingdaystomaturity_documentation_en-US" xlink:label="lab_efc_Assetssoldunderagreementstorepurchaseremainingdaystomaturity" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets sold under agreements to repurchase, remaining days to maturity</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Assetssoldunderagreementstorepurchaseremainingdaystomaturity" xlink:href="efc-20201231.xsd#efc_Assetssoldunderagreementstorepurchaseremainingdaystomaturity"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Assetssoldunderagreementstorepurchaseremainingdaystomaturity" xlink:to="lab_efc_Assetssoldunderagreementstorepurchaseremainingdaystomaturity" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MorganStanleyCollateralizedbyParValue1000ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member_67469dba-aaa6-4aea-b896-68fec2c16ab8_terseLabel_en-US" xlink:label="lab_efc_MorganStanleyCollateralizedbyParValue1000ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Morgan Stanley Collateralized by Par Value $1,000 Exchange-Traded Corporate Debt, Coupon 5.95%, Maturity Date 2026 12 [Member]</link:label>
    <link:label id="lab_efc_MorganStanleyCollateralizedbyParValue1000ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member_label_en-US" xlink:label="lab_efc_MorganStanleyCollateralizedbyParValue1000ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Morgan Stanley Collateralized by Par Value $1,000 Exchange-Traded Corporate Debt, Coupon 5.95%, Maturity Date 2026 12 [Member]</link:label>
    <link:label id="lab_efc_MorganStanleyCollateralizedbyParValue1000ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member_documentation_en-US" xlink:label="lab_efc_MorganStanleyCollateralizedbyParValue1000ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Morgan Stanley Collateralized by Par Value $1,000 Exchange-Traded Corporate Debt, Coupon 5.95%, Maturity Date 2026 12 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MorganStanleyCollateralizedbyParValue1000ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:href="efc-20201231.xsd#efc_MorganStanleyCollateralizedbyParValue1000ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MorganStanleyCollateralizedbyParValue1000ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:to="lab_efc_MorganStanleyCollateralizedbyParValue1000ExchangeTradedCorporateDebtCoupon5.95MaturityDate202612Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotSubjectToMasterNettingArrangement_7f1af426-8279-4e46-addd-fa48de6bf146_verboseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotSubjectToMasterNettingArrangement" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reverse repurchase agreements</link:label>
    <link:label id="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotSubjectToMasterNettingArrangement_b0e09a61-ebd8-42a1-ae77-23ccb2981356_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotSubjectToMasterNettingArrangement" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase agreements, at fair value</link:label>
    <link:label id="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotSubjectToMasterNettingArrangement_label_en-US" xlink:label="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotSubjectToMasterNettingArrangement" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Purchased under Agreements to Resell, Not Subject to Master Netting Arrangement</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotSubjectToMasterNettingArrangement" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotSubjectToMasterNettingArrangement"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotSubjectToMasterNettingArrangement" xlink:to="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellNotSubjectToMasterNettingArrangement" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ConsumerLoansDelinquencyStatusTableTextBlock_17356188-0ab4-4a2a-9d93-1eb77f2f08e8_terseLabel_en-US" xlink:label="lab_efc_ConsumerLoansDelinquencyStatusTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consumer Loans, Delinquency Status [Table Text Block]</link:label>
    <link:label id="lab_efc_ConsumerLoansDelinquencyStatusTableTextBlock_label_en-US" xlink:label="lab_efc_ConsumerLoansDelinquencyStatusTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consumer Loans, Delinquency Status [Table Text Block]</link:label>
    <link:label id="lab_efc_ConsumerLoansDelinquencyStatusTableTextBlock_documentation_en-US" xlink:label="lab_efc_ConsumerLoansDelinquencyStatusTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of consumer loans by delinquency status.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ConsumerLoansDelinquencyStatusTableTextBlock" xlink:href="efc-20201231.xsd#efc_ConsumerLoansDelinquencyStatusTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ConsumerLoansDelinquencyStatusTableTextBlock" xlink:to="lab_efc_ConsumerLoansDelinquencyStatusTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RestrictedCashAndInvestmentsAbstract_ab6ab44f-598e-4c06-92a5-64d3167361f9_terseLabel_en-US" xlink:label="lab_us-gaap_RestrictedCashAndInvestmentsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Restricted Cash and Investments [Abstract]</link:label>
    <link:label id="lab_us-gaap_RestrictedCashAndInvestmentsAbstract_label_en-US" xlink:label="lab_us-gaap_RestrictedCashAndInvestmentsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Restricted Cash and Investments [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RestrictedCashAndInvestmentsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RestrictedCashAndInvestmentsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RestrictedCashAndInvestmentsAbstract" xlink:to="lab_us-gaap_RestrictedCashAndInvestmentsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StockholdersEquityNoteAbstract_603fa914-2ef6-40cf-8c43-3b99abaf0b96_terseLabel_en-US" xlink:label="lab_us-gaap_StockholdersEquityNoteAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stockholders' Equity Note [Abstract]</link:label>
    <link:label id="lab_us-gaap_StockholdersEquityNoteAbstract_label_en-US" xlink:label="lab_us-gaap_StockholdersEquityNoteAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stockholders' Equity Note [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StockholdersEquityNoteAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockholdersEquityNoteAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StockholdersEquityNoteAbstract" xlink:to="lab_us-gaap_StockholdersEquityNoteAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_BorrowingsLineItems_8fcbac66-9ec3-45aa-9611-bd85eea76019_terseLabel_en-US" xlink:label="lab_efc_BorrowingsLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Borrowings [Line Items]</link:label>
    <link:label id="lab_efc_BorrowingsLineItems_label_en-US" xlink:label="lab_efc_BorrowingsLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Borrowings [Line Items]</link:label>
    <link:label id="lab_efc_BorrowingsLineItems_documentation_en-US" xlink:label="lab_efc_BorrowingsLineItems" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Line Items] for Borrowings [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_BorrowingsLineItems" xlink:href="efc-20201231.xsd#efc_BorrowingsLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_BorrowingsLineItems" xlink:to="lab_efc_BorrowingsLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RisksAndUncertaintiesAbstract_26ebaf73-d56f-44af-82b3-2d20c6411126_terseLabel_en-US" xlink:label="lab_us-gaap_RisksAndUncertaintiesAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Risks and Uncertainties [Abstract]</link:label>
    <link:label id="lab_us-gaap_RisksAndUncertaintiesAbstract_label_en-US" xlink:label="lab_us-gaap_RisksAndUncertaintiesAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Risks and Uncertainties [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RisksAndUncertaintiesAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RisksAndUncertaintiesAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RisksAndUncertaintiesAbstract" xlink:to="lab_us-gaap_RisksAndUncertaintiesAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Derivativefixedinterestrateincludingforwardstartingswaps_1440d4da-4287-4bd3-b979-bf1d2dd40a96_terseLabel_en-US" xlink:label="lab_efc_Derivativefixedinterestrateincludingforwardstartingswaps" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative fixed interest rate including forward-starting swaps</link:label>
    <link:label id="lab_efc_Derivativefixedinterestrateincludingforwardstartingswaps_label_en-US" xlink:label="lab_efc_Derivativefixedinterestrateincludingforwardstartingswaps" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative fixed interest rate including forward-starting swaps</link:label>
    <link:label id="lab_efc_Derivativefixedinterestrateincludingforwardstartingswaps_documentation_en-US" xlink:label="lab_efc_Derivativefixedinterestrateincludingforwardstartingswaps" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed interest rate related to the interest rate derivative, including forward-starting derivative.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Derivativefixedinterestrateincludingforwardstartingswaps" xlink:href="efc-20201231.xsd#efc_Derivativefixedinterestrateincludingforwardstartingswaps"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Derivativefixedinterestrateincludingforwardstartingswaps" xlink:to="lab_efc_Derivativefixedinterestrateincludingforwardstartingswaps" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LTIPUnitGrantDate_761be7cc-9500-4490-a368-eb79cbb9141a_terseLabel_en-US" xlink:label="lab_efc_LTIPUnitGrantDate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Grant Date</link:label>
    <link:label id="lab_efc_LTIPUnitGrantDate_label_en-US" xlink:label="lab_efc_LTIPUnitGrantDate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">LTIP Unit, Grant Date</link:label>
    <link:label id="lab_efc_LTIPUnitGrantDate_documentation_en-US" xlink:label="lab_efc_LTIPUnitGrantDate" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Date the LTIP unit was granted.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LTIPUnitGrantDate" xlink:href="efc-20201231.xsd#efc_LTIPUnitGrantDate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LTIPUnitGrantDate" xlink:to="lab_efc_LTIPUnitGrantDate" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Maturity181to360daysMember_790a1408-d36e-4d7b-853c-dc4d723dea6d_terseLabel_en-US" xlink:label="lab_efc_Maturity181to360daysMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">More Than 360 Days</link:label>
    <link:label id="lab_efc_Maturity181to360daysMember_label_en-US" xlink:label="lab_efc_Maturity181to360daysMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity 181 to 360 days [Member]</link:label>
    <link:label id="lab_efc_Maturity181to360daysMember_documentation_en-US" xlink:label="lab_efc_Maturity181to360daysMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity 181 to 360 days [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Maturity181to360daysMember" xlink:href="efc-20201231.xsd#efc_Maturity181to360daysMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Maturity181to360daysMember" xlink:to="lab_efc_Maturity181to360daysMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RecoverySwapMember_354cf890-2e75-4d32-b55d-ce3d2c4d86bb_terseLabel_en-US" xlink:label="lab_efc_RecoverySwapMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Recovery swaps</link:label>
    <link:label id="lab_efc_RecoverySwapMember_label_en-US" xlink:label="lab_efc_RecoverySwapMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Recovery Swap [Member]</link:label>
    <link:label id="lab_efc_RecoverySwapMember_documentation_en-US" xlink:label="lab_efc_RecoverySwapMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Receive a specified recovery rate in exchange for the actual recovery rate on the underlying</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RecoverySwapMember" xlink:href="efc-20201231.xsd#efc_RecoverySwapMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RecoverySwapMember" xlink:to="lab_efc_RecoverySwapMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MortgageBackedSecuritiesIssuedByUSGovernmentSponsoredEnterprisesMember_d27e50ed-dc65-4790-b0f1-474aafed5512_terseLabel_en-US" xlink:label="lab_us-gaap_MortgageBackedSecuritiesIssuedByUSGovernmentSponsoredEnterprisesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Agency RMBS</link:label>
    <link:label id="lab_us-gaap_MortgageBackedSecuritiesIssuedByUSGovernmentSponsoredEnterprisesMember_label_en-US" xlink:label="lab_us-gaap_MortgageBackedSecuritiesIssuedByUSGovernmentSponsoredEnterprisesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage-backed Securities, Issued by US Government Sponsored Enterprises [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MortgageBackedSecuritiesIssuedByUSGovernmentSponsoredEnterprisesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MortgageBackedSecuritiesIssuedByUSGovernmentSponsoredEnterprisesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MortgageBackedSecuritiesIssuedByUSGovernmentSponsoredEnterprisesMember" xlink:to="lab_us-gaap_MortgageBackedSecuritiesIssuedByUSGovernmentSponsoredEnterprisesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_TransactionTypeAxis_70161333-8b64-43cc-88a0-54dec6123b57_terseLabel_en-US" xlink:label="lab_us-gaap_TransactionTypeAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transaction Type [Axis]</link:label>
    <link:label id="lab_us-gaap_TransactionTypeAxis_label_en-US" xlink:label="lab_us-gaap_TransactionTypeAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transaction Type [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_TransactionTypeAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_TransactionTypeAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_TransactionTypeAxis" xlink:to="lab_us-gaap_TransactionTypeAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityFileNumber_0d5d9c9f-84c2-4422-b7ab-e831a64b3f32_terseLabel_en-US" xlink:label="lab_dei_EntityFileNumber" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity File Number</link:label>
    <link:label id="lab_dei_EntityFileNumber_label_en-US" xlink:label="lab_dei_EntityFileNumber" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity File Number</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityFileNumber" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityFileNumber"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityFileNumber" xlink:to="lab_dei_EntityFileNumber" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NetLIBOROASMember_5093dc68-dc05-41f2-9bc3-a3a46fc2b940_terseLabel_en-US" xlink:label="lab_efc_NetLIBOROASMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net LIBOR OAS [Member]</link:label>
    <link:label id="lab_efc_NetLIBOROASMember_label_en-US" xlink:label="lab_efc_NetLIBOROASMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net LIBOR OAS [Member]</link:label>
    <link:label id="lab_efc_NetLIBOROASMember_documentation_en-US" xlink:label="lab_efc_NetLIBOROASMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net LIBOR OAS (includes negative and positive OAS inputs) [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NetLIBOROASMember" xlink:href="efc-20201231.xsd#efc_NetLIBOROASMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NetLIBOROASMember" xlink:to="lab_efc_NetLIBOROASMember" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansOnRealEstateAbstract_label_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansOnRealEstateAbstract" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansOnRealEstateAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansOnRealEstateAbstract" xlink:to="lab_srt_MortgageLoansOnRealEstateAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EarningsPerShareBasicAndDilutedAbstract_4240a869-4532-4974-b339-957e179ad02c_terseLabel_en-US" xlink:label="lab_us-gaap_EarningsPerShareBasicAndDilutedAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Income (Loss) per Share of Common Stock:</link:label>
    <link:label id="lab_us-gaap_EarningsPerShareBasicAndDilutedAbstract_label_en-US" xlink:label="lab_us-gaap_EarningsPerShareBasicAndDilutedAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share, Basic and Diluted [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EarningsPerShareBasicAndDilutedAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EarningsPerShareBasicAndDilutedAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EarningsPerShareBasicAndDilutedAbstract" xlink:to="lab_us-gaap_EarningsPerShareBasicAndDilutedAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNet_e9d9b4a7-9176-4104-b535-3441430cfd96_terseLabel_en-US" xlink:label="lab_us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNet" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Impairment Losses, Recognized in Earnings</link:label>
    <link:label id="lab_us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNet_label_en-US" xlink:label="lab_us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other than Temporary Impairment Losses, Investments, Portion Recognized in Earnings, Net</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNet" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNet" xlink:to="lab_us-gaap_OtherThanTemporaryImpairmentLossesInvestmentsPortionRecognizedInEarningsNet" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PrincipalAndInterestFixedRateAgencySecuritiesMember_b8602094-a7ae-49ce-b312-02e8fdd59b32_terseLabel_en-US" xlink:label="lab_efc_PrincipalAndInterestFixedRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal And Interest Fixed Rate Agency Securities [Member]</link:label>
    <link:label id="lab_efc_PrincipalAndInterestFixedRateAgencySecuritiesMember_label_en-US" xlink:label="lab_efc_PrincipalAndInterestFixedRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal And Interest Fixed Rate Agency Securities [Member]</link:label>
    <link:label id="lab_efc_PrincipalAndInterestFixedRateAgencySecuritiesMember_documentation_en-US" xlink:label="lab_efc_PrincipalAndInterestFixedRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">A security issued by a government agency or government sponsored enterprise which makes principal and fixed interest rate payments.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PrincipalAndInterestFixedRateAgencySecuritiesMember" xlink:href="efc-20201231.xsd#efc_PrincipalAndInterestFixedRateAgencySecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PrincipalAndInterestFixedRateAgencySecuritiesMember" xlink:to="lab_efc_PrincipalAndInterestFixedRateAgencySecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleofFuturesContractsTableTextBlock_94215baa-8a75-495b-97dd-a1b012df8418_terseLabel_en-US" xlink:label="lab_efc_ScheduleofFuturesContractsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Futures Contracts [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleofFuturesContractsTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleofFuturesContractsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Futures Contracts [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleofFuturesContractsTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleofFuturesContractsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Tabular disclosure of futures contracts.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleofFuturesContractsTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleofFuturesContractsTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleofFuturesContractsTableTextBlock" xlink:to="lab_efc_ScheduleofFuturesContractsTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_WA_c9e96843-7e03-4848-8999-565dc5cde62f_terseLabel_en-US" xlink:label="lab_stpr_WA" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">MASSACHUSETTS</link:label>
    <link:label id="lab_stpr_WA_label_en-US" xlink:label="lab_stpr_WA" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">WASHINGTON</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_WA" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_WA"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_WA" xlink:to="lab_stpr_WA" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Expectedprincipalrepaymentsrelatedtoconsolidatedresidentialmortgageloansecuritizations_a39b28fc-32e3-420a-bb69-79ef4d30b0e1_terseLabel_en-US" xlink:label="lab_efc_Expectedprincipalrepaymentsrelatedtoconsolidatedresidentialmortgageloansecuritizations" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Expected principal repayments related to consolidated residential mortgage loan securitizations</link:label>
    <link:label id="lab_efc_Expectedprincipalrepaymentsrelatedtoconsolidatedresidentialmortgageloansecuritizations_label_en-US" xlink:label="lab_efc_Expectedprincipalrepaymentsrelatedtoconsolidatedresidentialmortgageloansecuritizations" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Expected principal repayments related to consolidated residential mortgage loan securitizations</link:label>
    <link:label id="lab_efc_Expectedprincipalrepaymentsrelatedtoconsolidatedresidentialmortgageloansecuritizations_documentation_en-US" xlink:label="lab_efc_Expectedprincipalrepaymentsrelatedtoconsolidatedresidentialmortgageloansecuritizations" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Expected principal repayments related to consolidated residential mortgage loan securitizations</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Expectedprincipalrepaymentsrelatedtoconsolidatedresidentialmortgageloansecuritizations" xlink:href="efc-20201231.xsd#efc_Expectedprincipalrepaymentsrelatedtoconsolidatedresidentialmortgageloansecuritizations"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Expectedprincipalrepaymentsrelatedtoconsolidatedresidentialmortgageloansecuritizations" xlink:to="lab_efc_Expectedprincipalrepaymentsrelatedtoconsolidatedresidentialmortgageloansecuritizations" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OtherInvestmentRelatedExpensesPolicyTextBlock_97e81b25-35db-42f8-bf06-decb89862b88_terseLabel_en-US" xlink:label="lab_efc_OtherInvestmentRelatedExpensesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Related Expenses</link:label>
    <link:label id="lab_efc_OtherInvestmentRelatedExpensesPolicyTextBlock_label_en-US" xlink:label="lab_efc_OtherInvestmentRelatedExpensesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Investment Related Expenses [Policy Text Block]</link:label>
    <link:label id="lab_efc_OtherInvestmentRelatedExpensesPolicyTextBlock_documentation_en-US" xlink:label="lab_efc_OtherInvestmentRelatedExpensesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The Company's policy for recognition of expenses directly related to specific financial instruments.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OtherInvestmentRelatedExpensesPolicyTextBlock" xlink:href="efc-20201231.xsd#efc_OtherInvestmentRelatedExpensesPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OtherInvestmentRelatedExpensesPolicyTextBlock" xlink:to="lab_efc_OtherInvestmentRelatedExpensesPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_DocumentFiscalPeriodFocus_0a65d8f6-46d5-462e-916c-3251d5225b8f_terseLabel_en-US" xlink:label="lab_dei_DocumentFiscalPeriodFocus" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Document Fiscal Period Focus</link:label>
    <link:label id="lab_dei_DocumentFiscalPeriodFocus_label_en-US" xlink:label="lab_dei_DocumentFiscalPeriodFocus" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Document Fiscal Period Focus</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_DocumentFiscalPeriodFocus" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_DocumentFiscalPeriodFocus"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_DocumentFiscalPeriodFocus" xlink:to="lab_dei_DocumentFiscalPeriodFocus" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_TransfersAndServicingOfFinancialInstrumentsTypesOfFinancialInstrumentsDomain_f9d95df9-b7d6-4420-977c-fbe54b3f6201_terseLabel_en-US" xlink:label="lab_us-gaap_TransfersAndServicingOfFinancialInstrumentsTypesOfFinancialInstrumentsDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instruments [Domain]</link:label>
    <link:label id="lab_us-gaap_TransfersAndServicingOfFinancialInstrumentsTypesOfFinancialInstrumentsDomain_label_en-US" xlink:label="lab_us-gaap_TransfersAndServicingOfFinancialInstrumentsTypesOfFinancialInstrumentsDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instruments [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_TransfersAndServicingOfFinancialInstrumentsTypesOfFinancialInstrumentsDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_TransfersAndServicingOfFinancialInstrumentsTypesOfFinancialInstrumentsDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_TransfersAndServicingOfFinancialInstrumentsTypesOfFinancialInstrumentsDomain" xlink:to="lab_us-gaap_TransfersAndServicingOfFinancialInstrumentsTypesOfFinancialInstrumentsDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_IncentiveFeeHurdleRateFloating_c00a640c-d837-4455-a20f-9d90b3c2f4ce_terseLabel_en-US" xlink:label="lab_efc_IncentiveFeeHurdleRateFloating" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Incentive fee hurdle rate floating</link:label>
    <link:label id="lab_efc_IncentiveFeeHurdleRateFloating_label_en-US" xlink:label="lab_efc_IncentiveFeeHurdleRateFloating" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Incentive fee hurdle rate floating</link:label>
    <link:label id="lab_efc_IncentiveFeeHurdleRateFloating_documentation_en-US" xlink:label="lab_efc_IncentiveFeeHurdleRateFloating" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage added onto the ten-year U.S. Treasury rate (for such fiscal quarter) to aggregate the floating hurdle rate used in the incentive fee calculation per the management agreement.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_IncentiveFeeHurdleRateFloating" xlink:href="efc-20201231.xsd#efc_IncentiveFeeHurdleRateFloating"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_IncentiveFeeHurdleRateFloating" xlink:to="lab_efc_IncentiveFeeHurdleRateFloating" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncreaseDecreaseInOtherAccountsPayableAndAccruedLiabilities_cba5c202-c803-432e-bc34-a7bc73287c5b_terseLabel_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInOtherAccountsPayableAndAccruedLiabilities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (decrease) in accrued expenses and other liabilities</link:label>
    <link:label id="lab_us-gaap_IncreaseDecreaseInOtherAccountsPayableAndAccruedLiabilities_label_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInOtherAccountsPayableAndAccruedLiabilities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (Decrease) in Other Accounts Payable and Accrued Liabilities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncreaseDecreaseInOtherAccountsPayableAndAccruedLiabilities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncreaseDecreaseInOtherAccountsPayableAndAccruedLiabilities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncreaseDecreaseInOtherAccountsPayableAndAccruedLiabilities" xlink:to="lab_us-gaap_IncreaseDecreaseInOtherAccountsPayableAndAccruedLiabilities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_UnrealizedGainLossonDerivativesIncludingGainLossfromForeignCurrencyRemeasurement_a1517eb3-01e8-4161-97d3-30a40a7876fa_terseLabel_en-US" xlink:label="lab_efc_UnrealizedGainLossonDerivativesIncludingGainLossfromForeignCurrencyRemeasurement" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized gains (losses) on financial derivatives, net</link:label>
    <link:label id="lab_efc_UnrealizedGainLossonDerivativesIncludingGainLossfromForeignCurrencyRemeasurement_label_en-US" xlink:label="lab_efc_UnrealizedGainLossonDerivativesIncludingGainLossfromForeignCurrencyRemeasurement" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized Gain (Loss) on Derivatives, Including Gain Loss from Foreign Currency Remeasurement</link:label>
    <link:label id="lab_efc_UnrealizedGainLossonDerivativesIncludingGainLossfromForeignCurrencyRemeasurement_documentation_en-US" xlink:label="lab_efc_UnrealizedGainLossonDerivativesIncludingGainLossfromForeignCurrencyRemeasurement" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized Gain (Loss) on Derivatives, Including Gain Loss from Foreign Currency Remeasurement</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_UnrealizedGainLossonDerivativesIncludingGainLossfromForeignCurrencyRemeasurement" xlink:href="efc-20201231.xsd#efc_UnrealizedGainLossonDerivativesIncludingGainLossfromForeignCurrencyRemeasurement"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_UnrealizedGainLossonDerivativesIncludingGainLossfromForeignCurrencyRemeasurement" xlink:to="lab_efc_UnrealizedGainLossonDerivativesIncludingGainLossfromForeignCurrencyRemeasurement" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ConsumerloansandassetbackedsecuritiesbackedbyconsumerloansMember_5db737b2-e6d9-4c90-bcec-ee3da7895f2e_terseLabel_en-US" xlink:label="lab_efc_ConsumerloansandassetbackedsecuritiesbackedbyconsumerloansMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consumer loans and asset-backed securities backed by consumer loans</link:label>
    <link:label id="lab_efc_ConsumerloansandassetbackedsecuritiesbackedbyconsumerloansMember_33d73415-9d63-4d8b-8875-0dedfa71a490_verboseLabel_en-US" xlink:label="lab_efc_ConsumerloansandassetbackedsecuritiesbackedbyconsumerloansMember" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consumer loans and asset-backed securities backed by consumer loans</link:label>
    <link:label id="lab_efc_ConsumerloansandassetbackedsecuritiesbackedbyconsumerloansMember_label_en-US" xlink:label="lab_efc_ConsumerloansandassetbackedsecuritiesbackedbyconsumerloansMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consumer loans and asset-backed securities backed by consumer loans [Member]</link:label>
    <link:label id="lab_efc_ConsumerloansandassetbackedsecuritiesbackedbyconsumerloansMember_documentation_en-US" xlink:label="lab_efc_ConsumerloansandassetbackedsecuritiesbackedbyconsumerloansMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consumer loans and asset-backed securities backed by consumer loans [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ConsumerloansandassetbackedsecuritiesbackedbyconsumerloansMember" xlink:href="efc-20201231.xsd#efc_ConsumerloansandassetbackedsecuritiesbackedbyconsumerloansMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ConsumerloansandassetbackedsecuritiesbackedbyconsumerloansMember" xlink:to="lab_efc_ConsumerloansandassetbackedsecuritiesbackedbyconsumerloansMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfDeferredTaxAssetsAndLiabilitiesTableTextBlock_582ea45f-92f9-4716-98a0-a93dae2f4fe3_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfDeferredTaxAssetsAndLiabilitiesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Deferred Tax Assets and Liabilities</link:label>
    <link:label id="lab_us-gaap_ScheduleOfDeferredTaxAssetsAndLiabilitiesTableTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfDeferredTaxAssetsAndLiabilitiesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Deferred Tax Assets and Liabilities [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfDeferredTaxAssetsAndLiabilitiesTableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfDeferredTaxAssetsAndLiabilitiesTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfDeferredTaxAssetsAndLiabilitiesTableTextBlock" xlink:to="lab_us-gaap_ScheduleOfDeferredTaxAssetsAndLiabilitiesTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PrincipalpaymentsonOthersecuredborrowingsatfairvaluenoncash_8895ca00-cc87-43b6-892a-635db9dbea4d_negatedTerseLabel_en-US" xlink:label="lab_efc_PrincipalpaymentsonOthersecuredborrowingsatfairvaluenoncash" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal payments on Other secured borrowings, at fair value (non-cash)</link:label>
    <link:label id="lab_efc_PrincipalpaymentsonOthersecuredborrowingsatfairvaluenoncash_label_en-US" xlink:label="lab_efc_PrincipalpaymentsonOthersecuredborrowingsatfairvaluenoncash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal payments on Other secured borrowings, at fair value (non-cash)</link:label>
    <link:label id="lab_efc_PrincipalpaymentsonOthersecuredborrowingsatfairvaluenoncash_documentation_en-US" xlink:label="lab_efc_PrincipalpaymentsonOthersecuredborrowingsatfairvaluenoncash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal payments on Other secured borrowings, at fair value (non-cash)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PrincipalpaymentsonOthersecuredborrowingsatfairvaluenoncash" xlink:href="efc-20201231.xsd#efc_PrincipalpaymentsonOthersecuredborrowingsatfairvaluenoncash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PrincipalpaymentsonOthersecuredborrowingsatfairvaluenoncash" xlink:to="lab_efc_PrincipalpaymentsonOthersecuredborrowingsatfairvaluenoncash" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansOnRealEstateFaceAmountOfMortgages_5871e3cc-e6d3-4b2f-a707-a6758b8deb1e_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateFaceAmountOfMortgages" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Face Amount of Mortgages</link:label>
    <link:label id="lab_srt_MortgageLoansOnRealEstateFaceAmountOfMortgages_label_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateFaceAmountOfMortgages" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Face Amount of Mortgages</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansOnRealEstateFaceAmountOfMortgages" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansOnRealEstateFaceAmountOfMortgages"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansOnRealEstateFaceAmountOfMortgages" xlink:to="lab_srt_MortgageLoansOnRealEstateFaceAmountOfMortgages" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfVariableInterestEntitiesTextBlock_5d0e28df-7c6f-481a-a72b-6681a057fec1_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfVariableInterestEntitiesTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Consolidated VIEs</link:label>
    <link:label id="lab_us-gaap_ScheduleOfVariableInterestEntitiesTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfVariableInterestEntitiesTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Variable Interest Entities [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfVariableInterestEntitiesTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfVariableInterestEntitiesTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfVariableInterestEntitiesTextBlock" xlink:to="lab_us-gaap_ScheduleOfVariableInterestEntitiesTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlements_f07541d2-3803-457f-9647-b3aaac7db3e4_terseLabel_en-US" xlink:label="lab_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlements" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Change in Net Unrealized Gains (Losses) Other Than on Accrued Periodic Settlements of Interest Rate Swaps</link:label>
    <link:label id="lab_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlements_label_en-US" xlink:label="lab_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlements" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Change in Net Unrealized Gains (Losses) on Accrued Periodic Settlements</link:label>
    <link:label id="lab_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlements_documentation_en-US" xlink:label="lab_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlements" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Change in Net Unrealized Gains (Losses) on Accrued Periodic Settlements</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlements" xlink:href="efc-20201231.xsd#efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlements"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlements" xlink:to="lab_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlements" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsTable_2a5c88c5-a037-4310-afff-afd99a845065_terseLabel_en-US" xlink:label="lab_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Organization, Consolidation and Presentation of Financial Statements [Table]</link:label>
    <link:label id="lab_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsTable_label_en-US" xlink:label="lab_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Organization, Consolidation and Presentation of Financial Statements [Table]</link:label>
    <link:label id="lab_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsTable_documentation_en-US" xlink:label="lab_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsTable" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Organization, Consolidation and Presentation of Financial Statements [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsTable" xlink:href="efc-20201231.xsd#efc_OrganizationConsolidationAndPresentationOfFinancialStatementsTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsTable" xlink:to="lab_efc_OrganizationConsolidationAndPresentationOfFinancialStatementsTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EuropeanSovereignBondMember_b56cc07b-7cf2-48f5-93bf-4768cf8f2a85_terseLabel_en-US" xlink:label="lab_efc_EuropeanSovereignBondMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">European Sovereign Bond [Member]</link:label>
    <link:label id="lab_efc_EuropeanSovereignBondMember_label_en-US" xlink:label="lab_efc_EuropeanSovereignBondMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">European Sovereign Bond [Member]</link:label>
    <link:label id="lab_efc_EuropeanSovereignBondMember_documentation_en-US" xlink:label="lab_efc_EuropeanSovereignBondMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">European Sovereign Bond [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EuropeanSovereignBondMember" xlink:href="efc-20201231.xsd#efc_EuropeanSovereignBondMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EuropeanSovereignBondMember" xlink:to="lab_efc_EuropeanSovereignBondMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_UnrealizedGainLossOnDerivatives_ac8ad959-9bd4-4721-b0a8-af010570092e_terseLabel_en-US" xlink:label="lab_us-gaap_UnrealizedGainLossOnDerivatives" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized gains (losses) on financial derivatives, net</link:label>
    <link:label id="lab_us-gaap_UnrealizedGainLossOnDerivatives_d6054924-0a0a-41a0-a218-9ab7d83cf135_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_UnrealizedGainLossOnDerivatives" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized (gains) losses on financial derivatives, net</link:label>
    <link:label id="lab_us-gaap_UnrealizedGainLossOnDerivatives_label_en-US" xlink:label="lab_us-gaap_UnrealizedGainLossOnDerivatives" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized Gain (Loss) on Derivatives</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_UnrealizedGainLossOnDerivatives" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_UnrealizedGainLossOnDerivatives"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_UnrealizedGainLossOnDerivatives" xlink:to="lab_us-gaap_UnrealizedGainLossOnDerivatives" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RestrictedCashAndCashEquivalentsItemsLineItems_9af5bb8a-f5e1-4f0a-8314-0435948f6586_terseLabel_en-US" xlink:label="lab_us-gaap_RestrictedCashAndCashEquivalentsItemsLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Restricted Cash [Line Items]</link:label>
    <link:label id="lab_us-gaap_RestrictedCashAndCashEquivalentsItemsLineItems_label_en-US" xlink:label="lab_us-gaap_RestrictedCashAndCashEquivalentsItemsLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Restricted Cash and Cash Equivalents Items [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RestrictedCashAndCashEquivalentsItemsLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RestrictedCashAndCashEquivalentsItemsLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RestrictedCashAndCashEquivalentsItemsLineItems" xlink:to="lab_us-gaap_RestrictedCashAndCashEquivalentsItemsLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LTIPUnitVestingDate_ad2281fc-15c0-4283-b398-6156d663ede0_terseLabel_en-US" xlink:label="lab_efc_LTIPUnitVestingDate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vesting Date(1)</link:label>
    <link:label id="lab_efc_LTIPUnitVestingDate_label_en-US" xlink:label="lab_efc_LTIPUnitVestingDate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">LTIP Unit, Vesting Date</link:label>
    <link:label id="lab_efc_LTIPUnitVestingDate_documentation_en-US" xlink:label="lab_efc_LTIPUnitVestingDate" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Date the LTIP unit vests.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LTIPUnitVestingDate" xlink:href="efc-20201231.xsd#efc_LTIPUnitVestingDate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LTIPUnitVestingDate" xlink:to="lab_efc_LTIPUnitVestingDate" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AccountsNotesAndLoansReceivableLineItems_bf89883d-995d-4435-b42b-99f8bd9993f3_terseLabel_en-US" xlink:label="lab_us-gaap_AccountsNotesAndLoansReceivableLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accounts, Notes, Loans and Financing Receivable [Line Items]</link:label>
    <link:label id="lab_us-gaap_AccountsNotesAndLoansReceivableLineItems_label_en-US" xlink:label="lab_us-gaap_AccountsNotesAndLoansReceivableLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accounts, Notes, Loans and Financing Receivable [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AccountsNotesAndLoansReceivableLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AccountsNotesAndLoansReceivableLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AccountsNotesAndLoansReceivableLineItems" xlink:to="lab_us-gaap_AccountsNotesAndLoansReceivableLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationFourPercentMember_da1729ac-7cdf-4621-ad40-3686010947c2_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Four Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationFourPercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Four Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationFourPercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Four Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationFourPercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationFourPercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationFourPercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationFourPercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_UnrealizedGainLossonOtherSecuredBorrowings_4cca658a-19d8-4b5f-bb5e-9404b381f262_terseLabel_en-US" xlink:label="lab_efc_UnrealizedGainLossonOtherSecuredBorrowings" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Secured Borrowings</link:label>
    <link:label id="lab_efc_UnrealizedGainLossonOtherSecuredBorrowings_label_en-US" xlink:label="lab_efc_UnrealizedGainLossonOtherSecuredBorrowings" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized Gain (Loss) on Other Secured Borrowings</link:label>
    <link:label id="lab_efc_UnrealizedGainLossonOtherSecuredBorrowings_documentation_en-US" xlink:label="lab_efc_UnrealizedGainLossonOtherSecuredBorrowings" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The net change in the difference between the fair value and the carrying value, or in the comparative fair values, of Other secured borrowings, held at each balance sheet date and included in earnings for the period.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_UnrealizedGainLossonOtherSecuredBorrowings" xlink:href="efc-20201231.xsd#efc_UnrealizedGainLossonOtherSecuredBorrowings"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_UnrealizedGainLossonOtherSecuredBorrowings" xlink:to="lab_efc_UnrealizedGainLossonOtherSecuredBorrowings" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncentiveFeeDescription_8e25e22d-34d5-4f9e-af42-cabf96ee57e0_terseLabel_en-US" xlink:label="lab_us-gaap_IncentiveFeeDescription" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Incentive fee, description</link:label>
    <link:label id="lab_us-gaap_IncentiveFeeDescription_label_en-US" xlink:label="lab_us-gaap_IncentiveFeeDescription" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Incentive Fee, Description</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncentiveFeeDescription" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncentiveFeeDescription"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncentiveFeeDescription" xlink:to="lab_us-gaap_IncentiveFeeDescription" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LongInvestmentHoldingsPercentOfEquityByRating_b809d10b-db88-4f2a-b279-ee298adc9cc6_terseLabel_en-US" xlink:label="lab_efc_LongInvestmentHoldingsPercentOfEquityByRating" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Holdings Percent Of Equity by Rating</link:label>
    <link:label id="lab_efc_LongInvestmentHoldingsPercentOfEquityByRating_label_en-US" xlink:label="lab_efc_LongInvestmentHoldingsPercentOfEquityByRating" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Holdings Percent Of Equity by Rating</link:label>
    <link:label id="lab_efc_LongInvestmentHoldingsPercentOfEquityByRating_documentation_en-US" xlink:label="lab_efc_LongInvestmentHoldingsPercentOfEquityByRating" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long investment holdings as a percentage of equity classified by rating</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LongInvestmentHoldingsPercentOfEquityByRating" xlink:href="efc-20201231.xsd#efc_LongInvestmentHoldingsPercentOfEquityByRating"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LongInvestmentHoldingsPercentOfEquityByRating" xlink:to="lab_efc_LongInvestmentHoldingsPercentOfEquityByRating" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_VA_4b08c029-b1c9-497c-b66f-eae8308d2e25_terseLabel_en-US" xlink:label="lab_stpr_VA" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Virginia [Member]</link:label>
    <link:label id="lab_stpr_VA_label_en-US" xlink:label="lab_stpr_VA" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">VIRGINIA</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_VA" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_VA"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_VA" xlink:to="lab_stpr_VA" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Advancesmadeunderwarehousefacility_be8c0c99-5c12-4368-a265-877d526b51c4_terseLabel_en-US" xlink:label="lab_efc_Advancesmadeunderwarehousefacility" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Advances made under warehouse facility</link:label>
    <link:label id="lab_efc_Advancesmadeunderwarehousefacility_label_en-US" xlink:label="lab_efc_Advancesmadeunderwarehousefacility" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Advances made under warehouse facility</link:label>
    <link:label id="lab_efc_Advancesmadeunderwarehousefacility_documentation_en-US" xlink:label="lab_efc_Advancesmadeunderwarehousefacility" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount outstanding and owed to the Company under a warehouse facility.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Advancesmadeunderwarehousefacility" xlink:href="efc-20201231.xsd#efc_Advancesmadeunderwarehousefacility"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Advancesmadeunderwarehousefacility" xlink:to="lab_efc_Advancesmadeunderwarehousefacility" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlementsofInterestRateSwaps_60a03ce3-dfbd-4be5-9a54-1aaeb981db29_terseLabel_en-US" xlink:label="lab_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlementsofInterestRateSwaps" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Change in Net Unrealized Gains (Losses) on Accrued Periodic Settlements of Interest Rate Swaps</link:label>
    <link:label id="lab_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlementsofInterestRateSwaps_label_en-US" xlink:label="lab_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlementsofInterestRateSwaps" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Change in Net Unrealized Gains (Losses) on Accrued Periodic Settlements of Interest Rate Swaps</link:label>
    <link:label id="lab_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlementsofInterestRateSwaps_documentation_en-US" xlink:label="lab_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlementsofInterestRateSwaps" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Change in Net Unrealized Gains (Losses) on Accrued Periodic Settlements of Interest Rate Swaps</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlementsofInterestRateSwaps" xlink:href="efc-20201231.xsd#efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlementsofInterestRateSwaps"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlementsofInterestRateSwaps" xlink:to="lab_efc_ChangeinNetUnrealizedGainsLossesonAccruedPeriodicSettlementsofInterestRateSwaps" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StatementLineItems_1ffab8a2-6045-43cc-859d-a59debb145c3_terseLabel_en-US" xlink:label="lab_us-gaap_StatementLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Statement [Line Items]</link:label>
    <link:label id="lab_us-gaap_StatementLineItems_label_en-US" xlink:label="lab_us-gaap_StatementLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Statement [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StatementLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StatementLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StatementLineItems" xlink:to="lab_us-gaap_StatementLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ProceedsFromBrokersInvestingActivities_3b4abf73-5d90-4c09-9bce-40a9a3d1579c_terseLabel_en-US" xlink:label="lab_efc_ProceedsFromBrokersInvestingActivities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Due from brokers, net</link:label>
    <link:label id="lab_efc_ProceedsFromBrokersInvestingActivities_label_en-US" xlink:label="lab_efc_ProceedsFromBrokersInvestingActivities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds From Brokers, Investing Activities</link:label>
    <link:label id="lab_efc_ProceedsFromBrokersInvestingActivities_documentation_en-US" xlink:label="lab_efc_ProceedsFromBrokersInvestingActivities" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds From Brokers, Investing Activities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ProceedsFromBrokersInvestingActivities" xlink:href="efc-20201231.xsd#efc_ProceedsFromBrokersInvestingActivities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ProceedsFromBrokersInvestingActivities" xlink:to="lab_efc_ProceedsFromBrokersInvestingActivities" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ManagementFeePayable_326ac738-df96-461a-8bed-094367152db1_terseLabel_en-US" xlink:label="lab_us-gaap_ManagementFeePayable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Base management fee payable to affiliate</link:label>
    <link:label id="lab_us-gaap_ManagementFeePayable_label_en-US" xlink:label="lab_us-gaap_ManagementFeePayable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Management Fee Payable</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ManagementFeePayable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ManagementFeePayable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ManagementFeePayable" xlink:to="lab_us-gaap_ManagementFeePayable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Proceedsfromfinancialderivatives_9ead93ae-08f0-4a27-9789-0d33e5e33360_terseLabel_en-US" xlink:label="lab_efc_Proceedsfromfinancialderivatives" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from financial derivatives</link:label>
    <link:label id="lab_efc_Proceedsfromfinancialderivatives_label_en-US" xlink:label="lab_efc_Proceedsfromfinancialderivatives" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from financial derivatives</link:label>
    <link:label id="lab_efc_Proceedsfromfinancialderivatives_documentation_en-US" xlink:label="lab_efc_Proceedsfromfinancialderivatives" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from financial derivatives</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Proceedsfromfinancialderivatives" xlink:href="efc-20201231.xsd#efc_Proceedsfromfinancialderivatives"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Proceedsfromfinancialderivatives" xlink:to="lab_efc_Proceedsfromfinancialderivatives" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentCompanyInvestmentIncomeLossFromOperationsPerShare_6889aff1-2959-4a97-bc4b-7896ee1ba93d_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentCompanyInvestmentIncomeLossFromOperationsPerShare" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Results of Operations Attributable to Equity</link:label>
    <link:label id="lab_us-gaap_InvestmentCompanyInvestmentIncomeLossFromOperationsPerShare_label_en-US" xlink:label="lab_us-gaap_InvestmentCompanyInvestmentIncomeLossFromOperationsPerShare" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Investment Income (Loss) from Operations, Per Share</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentCompanyInvestmentIncomeLossFromOperationsPerShare" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentCompanyInvestmentIncomeLossFromOperationsPerShare"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentCompanyInvestmentIncomeLossFromOperationsPerShare" xlink:to="lab_us-gaap_InvestmentCompanyInvestmentIncomeLossFromOperationsPerShare" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_BorrowingsTable_2e5cedfc-ffc7-4008-9ee8-5bd7da9b33b1_terseLabel_en-US" xlink:label="lab_efc_BorrowingsTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Borrowings [Table]</link:label>
    <link:label id="lab_efc_BorrowingsTable_label_en-US" xlink:label="lab_efc_BorrowingsTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Borrowings [Table]</link:label>
    <link:label id="lab_efc_BorrowingsTable_documentation_en-US" xlink:label="lab_efc_BorrowingsTable" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Borrowings [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_BorrowingsTable" xlink:href="efc-20201231.xsd#efc_BorrowingsTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_BorrowingsTable" xlink:to="lab_efc_BorrowingsTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueDisclosuresAbstract_3fd7be82-d831-4f8a-a8c7-43d1bcad4d33_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueDisclosuresAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Disclosures [Abstract]</link:label>
    <link:label id="lab_us-gaap_FairValueDisclosuresAbstract_label_en-US" xlink:label="lab_us-gaap_FairValueDisclosuresAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Disclosures [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueDisclosuresAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueDisclosuresAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueDisclosuresAbstract" xlink:to="lab_us-gaap_FairValueDisclosuresAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CommonStockDividendsPerShareDeclared_6ec5a290-e1f8-474e-bc55-d90efc509776_terseLabel_en-US" xlink:label="lab_us-gaap_CommonStockDividendsPerShareDeclared" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends declared (in usd per share)</link:label>
    <link:label id="lab_us-gaap_CommonStockDividendsPerShareDeclared_label_en-US" xlink:label="lab_us-gaap_CommonStockDividendsPerShareDeclared" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common Stock, Dividends, Per Share, Declared</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CommonStockDividendsPerShareDeclared" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CommonStockDividendsPerShareDeclared"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CommonStockDividendsPerShareDeclared" xlink:to="lab_us-gaap_CommonStockDividendsPerShareDeclared" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_TotalUnrealizedGainLoss_cb2d5352-f533-48cd-8aed-64d3e18bfee7_terseLabel_en-US" xlink:label="lab_efc_TotalUnrealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Unrealized Gain (Loss)</link:label>
    <link:label id="lab_efc_TotalUnrealizedGainLoss_label_en-US" xlink:label="lab_efc_TotalUnrealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Unrealized Gain (Loss)</link:label>
    <link:label id="lab_efc_TotalUnrealizedGainLoss_documentation_en-US" xlink:label="lab_efc_TotalUnrealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The net change in the difference between the fair value and the carrying value, or in the comparative fair values, of investments, other secured borrowings, derivative instruments, held at each balance sheet date, and foreign currency translation, that was included in earnings for the period.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_TotalUnrealizedGainLoss" xlink:href="efc-20201231.xsd#efc_TotalUnrealizedGainLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_TotalUnrealizedGainLoss" xlink:to="lab_efc_TotalUnrealizedGainLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EarningsPerSharePolicyTextBlock_a417510a-c991-440d-bd4c-8c533365f3f0_verboseLabel_en-US" xlink:label="lab_us-gaap_EarningsPerSharePolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share ("EPS")</link:label>
    <link:label id="lab_us-gaap_EarningsPerSharePolicyTextBlock_label_en-US" xlink:label="lab_us-gaap_EarningsPerSharePolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share, Policy [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EarningsPerSharePolicyTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EarningsPerSharePolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EarningsPerSharePolicyTextBlock" xlink:to="lab_us-gaap_EarningsPerSharePolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansOnRealEstateByLoanDisclosureTextBlock_75a8e78d-a3c4-449d-a89a-314af2e795d3_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateByLoanDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate</link:label>
    <link:label id="lab_srt_MortgageLoansOnRealEstateByLoanDisclosureTextBlock_label_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateByLoanDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansOnRealEstateByLoanDisclosureTextBlock" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansOnRealEstateByLoanDisclosureTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansOnRealEstateByLoanDisclosureTextBlock" xlink:to="lab_srt_MortgageLoansOnRealEstateByLoanDisclosureTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ConcentrationRiskTreshold_029d01f2-89bb-4b9f-b898-47be5c661161_terseLabel_en-US" xlink:label="lab_efc_ConcentrationRiskTreshold" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration risk threshold</link:label>
    <link:label id="lab_efc_ConcentrationRiskTreshold_label_en-US" xlink:label="lab_efc_ConcentrationRiskTreshold" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk Treshold</link:label>
    <link:label id="lab_efc_ConcentrationRiskTreshold_documentation_en-US" xlink:label="lab_efc_ConcentrationRiskTreshold" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Threshold used by the Company to determine disclosure of counterparty concentration risk.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ConcentrationRiskTreshold" xlink:href="efc-20201231.xsd#efc_ConcentrationRiskTreshold"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ConcentrationRiskTreshold" xlink:to="lab_efc_ConcentrationRiskTreshold" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_UT_d7c0a806-627e-4075-bbc4-ca0902aee2de_terseLabel_en-US" xlink:label="lab_stpr_UT" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">UTAH</link:label>
    <link:label id="lab_stpr_UT_label_en-US" xlink:label="lab_stpr_UT" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">UTAH</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_UT" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_UT"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_UT" xlink:to="lab_stpr_UT" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1500000And1749999Member_66345348-7aed-4f34-afdd-50ce3ea4e146_terseLabel_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1500000And1749999Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $1,500,000 and $1,749,999</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1500000And1749999Member_label_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1500000And1749999Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $1,500,000 and $1,749,999 [Member]</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1500000And1749999Member_documentation_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1500000And1749999Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $1,500,000 and $1,749,999</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1500000And1749999Member" xlink:href="efc-20201231.xsd#efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1500000And1749999Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1500000And1749999Member" xlink:to="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween1500000And1749999Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedGainBeforeTax_7579c08d-9dfb-4c68-9804-55b04c73f40a_terseLabel_en-US" xlink:label="lab_us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedGainBeforeTax" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate equity, gross unrealized gains</link:label>
    <link:label id="lab_us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedGainBeforeTax_label_en-US" xlink:label="lab_us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedGainBeforeTax" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Available-for-sale Equity Securities, Accumulated Gross Unrealized Gain, before Tax</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedGainBeforeTax" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedGainBeforeTax"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedGainBeforeTax" xlink:to="lab_us-gaap_AvailableForSaleEquitySecuritiesAccumulatedGrossUnrealizedGainBeforeTax" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeAssetFairValueGrossLiability_d2fb5e39-38eb-4fb0-83b6-1e33a01ed254_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeAssetFairValueGrossLiability" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Asset, Fair Value, Gross Liability</link:label>
    <link:label id="lab_us-gaap_DerivativeAssetFairValueGrossLiability_label_en-US" xlink:label="lab_us-gaap_DerivativeAssetFairValueGrossLiability" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Asset, Fair Value, Gross Liability</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeAssetFairValueGrossLiability" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeAssetFairValueGrossLiability"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeAssetFairValueGrossLiability" xlink:to="lab_us-gaap_DerivativeAssetFairValueGrossLiability" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_WhollyOwnedPropertiesMember_77fbb7a6-21db-4629-be4e-dc13bdbc9b94_terseLabel_en-US" xlink:label="lab_us-gaap_WhollyOwnedPropertiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real estate owned</link:label>
    <link:label id="lab_us-gaap_WhollyOwnedPropertiesMember_98aab5b8-1bf2-4406-9ca9-371f58496eac_verboseLabel_en-US" xlink:label="lab_us-gaap_WhollyOwnedPropertiesMember" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real estate owned</link:label>
    <link:label id="lab_us-gaap_WhollyOwnedPropertiesMember_label_en-US" xlink:label="lab_us-gaap_WhollyOwnedPropertiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Wholly Owned Properties [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_WhollyOwnedPropertiesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_WhollyOwnedPropertiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_WhollyOwnedPropertiesMember" xlink:to="lab_us-gaap_WhollyOwnedPropertiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NetCashProvidedByUsedInOperatingActivitiesContinuingOperationsAbstract_4f9eb3bb-7a97-4cba-99c0-004985c153b4_terseLabel_en-US" xlink:label="lab_us-gaap_NetCashProvidedByUsedInOperatingActivitiesContinuingOperationsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash flows provided by (used in) operating activities:</link:label>
    <link:label id="lab_us-gaap_NetCashProvidedByUsedInOperatingActivitiesContinuingOperationsAbstract_label_en-US" xlink:label="lab_us-gaap_NetCashProvidedByUsedInOperatingActivitiesContinuingOperationsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Cash Provided by (Used in) Operating Activities, Continuing Operations [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NetCashProvidedByUsedInOperatingActivitiesContinuingOperationsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NetCashProvidedByUsedInOperatingActivitiesContinuingOperationsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NetCashProvidedByUsedInOperatingActivitiesContinuingOperationsAbstract" xlink:to="lab_us-gaap_NetCashProvidedByUsedInOperatingActivitiesContinuingOperationsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MinorityInterestLineItems_ed407b53-e038-46ba-8323-f87064da7edc_terseLabel_en-US" xlink:label="lab_us-gaap_MinorityInterestLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noncontrolling Interest [Line Items]</link:label>
    <link:label id="lab_us-gaap_MinorityInterestLineItems_label_en-US" xlink:label="lab_us-gaap_MinorityInterestLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noncontrolling Interest [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MinorityInterestLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MinorityInterestLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MinorityInterestLineItems" xlink:to="lab_us-gaap_MinorityInterestLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SharesIssuedAsPaymentofIncentiveFee_9cb9e0ca-ecf8-4bec-9852-999f5d4a468c_terseLabel_en-US" xlink:label="lab_efc_SharesIssuedAsPaymentofIncentiveFee" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Shares Issued As Payment of Incentive Fee</link:label>
    <link:label id="lab_efc_SharesIssuedAsPaymentofIncentiveFee_label_en-US" xlink:label="lab_efc_SharesIssuedAsPaymentofIncentiveFee" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Shares Issued As Payment of Incentive Fee</link:label>
    <link:label id="lab_efc_SharesIssuedAsPaymentofIncentiveFee_documentation_en-US" xlink:label="lab_efc_SharesIssuedAsPaymentofIncentiveFee" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Value of shares issued to affiliate as partial payment for performance fees.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SharesIssuedAsPaymentofIncentiveFee" xlink:href="efc-20201231.xsd#efc_SharesIssuedAsPaymentofIncentiveFee"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SharesIssuedAsPaymentofIncentiveFee" xlink:to="lab_efc_SharesIssuedAsPaymentofIncentiveFee" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_AmendmentFlag_677048d7-2b52-474e-8e4c-bba095a74eae_terseLabel_en-US" xlink:label="lab_dei_AmendmentFlag" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amendment Flag</link:label>
    <link:label id="lab_dei_AmendmentFlag_label_en-US" xlink:label="lab_dei_AmendmentFlag" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amendment Flag</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_AmendmentFlag" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_AmendmentFlag"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_AmendmentFlag" xlink:to="lab_dei_AmendmentFlag" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OperatingPartnershipLTIPUnits_7cdb2e1a-17e1-4a9e-95b4-7849755dee8d_terseLabel_en-US" xlink:label="lab_efc_OperatingPartnershipLTIPUnits" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership LTIP Units (in units)</link:label>
    <link:label id="lab_efc_OperatingPartnershipLTIPUnits_label_en-US" xlink:label="lab_efc_OperatingPartnershipLTIPUnits" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership LTIP Units</link:label>
    <link:label id="lab_efc_OperatingPartnershipLTIPUnits_documentation_en-US" xlink:label="lab_efc_OperatingPartnershipLTIPUnits" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership LTIP Units</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OperatingPartnershipLTIPUnits" xlink:href="efc-20201231.xsd#efc_OperatingPartnershipLTIPUnits"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OperatingPartnershipLTIPUnits" xlink:to="lab_efc_OperatingPartnershipLTIPUnits" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureCapitalExpendituresAndOtherAdjustments_a649cf24-85c2-432b-a29f-f39d35966a1c_terseLabel_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureCapitalExpendituresAndOtherAdjustments" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Capital expenditures and other adjustments to cost</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureCapitalExpendituresAndOtherAdjustments_label_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureCapitalExpendituresAndOtherAdjustments" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure, Capital Expenditures And Other Adjustments</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureCapitalExpendituresAndOtherAdjustments_documentation_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureCapitalExpendituresAndOtherAdjustments" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure, Capital Expenditures And Other Adjustments</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RealEstateAcquiredThroughForeclosureCapitalExpendituresAndOtherAdjustments" xlink:href="efc-20201231.xsd#efc_RealEstateAcquiredThroughForeclosureCapitalExpendituresAndOtherAdjustments"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RealEstateAcquiredThroughForeclosureCapitalExpendituresAndOtherAdjustments" xlink:to="lab_efc_RealEstateAcquiredThroughForeclosureCapitalExpendituresAndOtherAdjustments" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SharesRepurchasedPolicyPolicyTextBlock_f11c0d38-dfd2-4f9f-81fc-398b067cffe3_terseLabel_en-US" xlink:label="lab_efc_SharesRepurchasedPolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Shares Repurchased</link:label>
    <link:label id="lab_efc_SharesRepurchasedPolicyPolicyTextBlock_label_en-US" xlink:label="lab_efc_SharesRepurchasedPolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Shares Repurchased Policy [Policy Text Block]</link:label>
    <link:label id="lab_efc_SharesRepurchasedPolicyPolicyTextBlock_documentation_en-US" xlink:label="lab_efc_SharesRepurchasedPolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disclosure of accounting policy regarding the treatment of repurchased common shares.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SharesRepurchasedPolicyPolicyTextBlock" xlink:href="efc-20201231.xsd#efc_SharesRepurchasedPolicyPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SharesRepurchasedPolicyPolicyTextBlock" xlink:to="lab_efc_SharesRepurchasedPolicyPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NumberofCounterpartieswithOutstandingReverseRepurchaseAgreements_5025706b-b590-4071-8316-8881ca1ac494_terseLabel_en-US" xlink:label="lab_efc_NumberofCounterpartieswithOutstandingReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of Counterparties with Outstanding Reverse Repurchase Agreements</link:label>
    <link:label id="lab_efc_NumberofCounterpartieswithOutstandingReverseRepurchaseAgreements_label_en-US" xlink:label="lab_efc_NumberofCounterpartieswithOutstandingReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of Counterparties with Outstanding Reverse Repurchase Agreements</link:label>
    <link:label id="lab_efc_NumberofCounterpartieswithOutstandingReverseRepurchaseAgreements_documentation_en-US" xlink:label="lab_efc_NumberofCounterpartieswithOutstandingReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of Counterparties with Outstanding Reverse Repurchase Agreements</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NumberofCounterpartieswithOutstandingReverseRepurchaseAgreements" xlink:href="efc-20201231.xsd#efc_NumberofCounterpartieswithOutstandingReverseRepurchaseAgreements"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NumberofCounterpartieswithOutstandingReverseRepurchaseAgreements" xlink:to="lab_efc_NumberofCounterpartieswithOutstandingReverseRepurchaseAgreements" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_TX_aee29e71-7f64-4c6f-847c-caeb97e2816c_terseLabel_en-US" xlink:label="lab_stpr_TX" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Texas [Member]</link:label>
    <link:label id="lab_stpr_TX_label_en-US" xlink:label="lab_stpr_TX" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">TEXAS</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_TX" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_TX"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_TX" xlink:to="lab_stpr_TX" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DividendsPreferredStock_1bad925c-0592-450d-9bfc-0b7dfb802c0d_terseLabel_en-US" xlink:label="lab_us-gaap_DividendsPreferredStock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends on preferred stock</link:label>
    <link:label id="lab_us-gaap_DividendsPreferredStock_label_en-US" xlink:label="lab_us-gaap_DividendsPreferredStock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends, Preferred Stock</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DividendsPreferredStock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DividendsPreferredStock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DividendsPreferredStock" xlink:to="lab_us-gaap_DividendsPreferredStock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CommonStockSharesOutstanding_22cd28dd-f6fc-4d13-8ed4-ab1df00ef721_terseLabel_en-US" xlink:label="lab_us-gaap_CommonStockSharesOutstanding" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common stock, shares outstanding</link:label>
    <link:label id="lab_us-gaap_CommonStockSharesOutstanding_741fee59-ce20-4ac6-99e8-edc96ba42a2f_periodStartLabel_en-US" xlink:label="lab_us-gaap_CommonStockSharesOutstanding" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Beginning balance (in shares)</link:label>
    <link:label id="lab_us-gaap_CommonStockSharesOutstanding_89ecd498-0165-4bba-a2d7-703beaf35213_periodEndLabel_en-US" xlink:label="lab_us-gaap_CommonStockSharesOutstanding" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ending balance (in shares)</link:label>
    <link:label id="lab_us-gaap_CommonStockSharesOutstanding_label_en-US" xlink:label="lab_us-gaap_CommonStockSharesOutstanding" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common Stock, Shares, Outstanding</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CommonStockSharesOutstanding" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CommonStockSharesOutstanding"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CommonStockSharesOutstanding" xlink:to="lab_us-gaap_CommonStockSharesOutstanding" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FutureMember_695f25d9-dbe7-4d57-9465-f57044deaba7_terseLabel_en-US" xlink:label="lab_us-gaap_FutureMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Futures</link:label>
    <link:label id="lab_us-gaap_FutureMember_label_en-US" xlink:label="lab_us-gaap_FutureMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Future [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FutureMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FutureMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FutureMember" xlink:to="lab_us-gaap_FutureMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202111MemberDomain_418b180c-0265-4166-8888-31df512cb214_terseLabel_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202111MemberDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.88 Percent, Maturity Date 2021 11 [Member] [Domain]</link:label>
    <link:label id="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202111MemberDomain_label_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202111MemberDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.88 Percent, Maturity Date 2021 11 [Member] [Domain]</link:label>
    <link:label id="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202111MemberDomain_documentation_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202111MemberDomain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.88 Percent, Maturity Date 2021 11 [Member] [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_U.S.TreasuryNote2.88PercentMaturityDate202111MemberDomain" xlink:href="efc-20201231.xsd#efc_U.S.TreasuryNote2.88PercentMaturityDate202111MemberDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_U.S.TreasuryNote2.88PercentMaturityDate202111MemberDomain" xlink:to="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202111MemberDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansOnRealEstateInterestRate_518f38b8-f2ca-4c79-a1c9-3b8a1a56a937_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateInterestRate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Interest Rate</link:label>
    <link:label id="lab_srt_MortgageLoansOnRealEstateInterestRate_label_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateInterestRate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Interest Rate</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansOnRealEstateInterestRate" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansOnRealEstateInterestRate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansOnRealEstateInterestRate" xlink:to="lab_srt_MortgageLoansOnRealEstateInterestRate" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_UsGaap_MeasurementInputDiscountRateIncludingNegativeRateMemberMember_a29f83bb-2767-449a-995f-bef81e3360cb_terseLabel_en-US" xlink:label="lab_efc_UsGaap_MeasurementInputDiscountRateIncludingNegativeRateMemberMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">us-gaap_MeasurementInputDiscountRateIncludingNegativeRateMember [Member]</link:label>
    <link:label id="lab_efc_UsGaap_MeasurementInputDiscountRateIncludingNegativeRateMemberMember_label_en-US" xlink:label="lab_efc_UsGaap_MeasurementInputDiscountRateIncludingNegativeRateMemberMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">us-gaap_MeasurementInputDiscountRateIncludingNegativeRateMember [Member]</link:label>
    <link:label id="lab_efc_UsGaap_MeasurementInputDiscountRateIncludingNegativeRateMemberMember_documentation_en-US" xlink:label="lab_efc_UsGaap_MeasurementInputDiscountRateIncludingNegativeRateMemberMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">us-gaap_MeasurementInputDiscountRateIncludingNegativeRateMember</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_UsGaap_MeasurementInputDiscountRateIncludingNegativeRateMemberMember" xlink:href="efc-20201231.xsd#efc_UsGaap_MeasurementInputDiscountRateIncludingNegativeRateMemberMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_UsGaap_MeasurementInputDiscountRateIncludingNegativeRateMemberMember" xlink:to="lab_efc_UsGaap_MeasurementInputDiscountRateIncludingNegativeRateMemberMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueInputsLevel1Member_be770f73-71a0-4b2e-87da-189fdc9e5825_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueInputsLevel1Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Level 1</link:label>
    <link:label id="lab_us-gaap_FairValueInputsLevel1Member_label_en-US" xlink:label="lab_us-gaap_FairValueInputsLevel1Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Inputs, Level 1 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueInputsLevel1Member" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueInputsLevel1Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueInputsLevel1Member" xlink:to="lab_us-gaap_FairValueInputsLevel1Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsTwentyYearFourPointFivePercentMember_bef65fa8-ed9b-4ed6-b3cf-5bc5f716c0ee_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsTwentyYearFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Twenty Year) Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsTwentyYearFourPointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsTwentyYearFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Twenty Year) Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsTwentyYearFourPointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsTwentyYearFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Twenty Year) Four Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationPoolsTwentyYearFourPointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationPoolsTwentyYearFourPointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationPoolsTwentyYearFourPointFivePercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationPoolsTwentyYearFourPointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DerivativeMaturityPeriodDomain_32293937-b2dc-4d89-b066-0ff3bbd88521_terseLabel_en-US" xlink:label="lab_efc_DerivativeMaturityPeriodDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Maturity Period [Domain]</link:label>
    <link:label id="lab_efc_DerivativeMaturityPeriodDomain_label_en-US" xlink:label="lab_efc_DerivativeMaturityPeriodDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Maturity Period [Domain]</link:label>
    <link:label id="lab_efc_DerivativeMaturityPeriodDomain_documentation_en-US" xlink:label="lab_efc_DerivativeMaturityPeriodDomain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Domain] for Derivative Maturity Period [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DerivativeMaturityPeriodDomain" xlink:href="efc-20201231.xsd#efc_DerivativeMaturityPeriodDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DerivativeMaturityPeriodDomain" xlink:to="lab_efc_DerivativeMaturityPeriodDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_DirectorMember_0e5c5615-80d3-42fc-891c-695f9d37bd7c_terseLabel_en-US" xlink:label="lab_srt_DirectorMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Director [Member]</link:label>
    <link:label id="lab_srt_DirectorMember_label_en-US" xlink:label="lab_srt_DirectorMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Director [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_DirectorMember" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_DirectorMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_DirectorMember" xlink:to="lab_srt_DirectorMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StockholdersEquityNoteDisclosureTextBlock_cc18936c-3a3c-41c8-9350-ad7beef46fa3_verboseLabel_en-US" xlink:label="lab_us-gaap_StockholdersEquityNoteDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common Stock Capitalization</link:label>
    <link:label id="lab_us-gaap_StockholdersEquityNoteDisclosureTextBlock_label_en-US" xlink:label="lab_us-gaap_StockholdersEquityNoteDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stockholders' Equity Note Disclosure [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StockholdersEquityNoteDisclosureTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockholdersEquityNoteDisclosureTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StockholdersEquityNoteDisclosureTextBlock" xlink:to="lab_us-gaap_StockholdersEquityNoteDisclosureTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_TransferToOtherRealEstate_e2fe90d7-7029-4cfe-bb5a-fbf2129e51c8_terseLabel_en-US" xlink:label="lab_us-gaap_TransferToOtherRealEstate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transfers from mortgage loans to real estate owned (non-cash)</link:label>
    <link:label id="lab_us-gaap_TransferToOtherRealEstate_47fbfeb5-a335-47e2-bbe0-1690334a1142_verboseLabel_en-US" xlink:label="lab_us-gaap_TransferToOtherRealEstate" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transfers from mortgage loans</link:label>
    <link:label id="lab_us-gaap_TransferToOtherRealEstate_label_en-US" xlink:label="lab_us-gaap_TransferToOtherRealEstate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Owned, Transfer to Real Estate Owned</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_TransferToOtherRealEstate" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_TransferToOtherRealEstate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_TransferToOtherRealEstate" xlink:to="lab_us-gaap_TransferToOtherRealEstate" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_Dividends_09c5313c-cf90-4e16-a913-41894cc85691_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_Dividends" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends</link:label>
    <link:label id="lab_us-gaap_Dividends_label_en-US" xlink:label="lab_us-gaap_Dividends" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_Dividends" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_Dividends"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_Dividends" xlink:to="lab_us-gaap_Dividends" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_IcfrAuditorAttestationFlag_d2614413-490f-4cd1-9c88-caf674953ef3_terseLabel_en-US" xlink:label="lab_efc_IcfrAuditorAttestationFlag" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">IcfrAuditorAttestationFlag</link:label>
    <link:label id="lab_efc_IcfrAuditorAttestationFlag_label_en-US" xlink:label="lab_efc_IcfrAuditorAttestationFlag" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">IcfrAuditorAttestationFlag</link:label>
    <link:label id="lab_efc_IcfrAuditorAttestationFlag_documentation_en-US" xlink:label="lab_efc_IcfrAuditorAttestationFlag" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">IcfrAuditorAttestationFlag</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_IcfrAuditorAttestationFlag" xlink:href="efc-20201231.xsd#efc_IcfrAuditorAttestationFlag"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_IcfrAuditorAttestationFlag" xlink:to="lab_efc_IcfrAuditorAttestationFlag" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_TN_f5561cd2-3200-49c1-959f-4f750d8a0e96_terseLabel_en-US" xlink:label="lab_stpr_TN" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Tennessee [Member]</link:label>
    <link:label id="lab_stpr_TN_label_en-US" xlink:label="lab_stpr_TN" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">TENNESSEE</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_TN" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_TN"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_TN" xlink:to="lab_stpr_TN" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPosition_ecaf8aa3-d223-4bc3-a250-56f11d3d9b47_terseLabel_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPosition" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Total</link:label>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPosition_label_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPosition" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Unrealized Loss Position</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPosition" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPosition"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPosition" xlink:to="lab_us-gaap_DebtSecuritiesAvailableForSaleUnrealizedLossPosition" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ElizonTCGSBC20171Member_9e6ee16b-fad0-4bbd-a91d-fc0170af212a_terseLabel_en-US" xlink:label="lab_efc_ElizonTCGSBC20171Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Elizon TCG SBC 2017-1</link:label>
    <link:label id="lab_efc_ElizonTCGSBC20171Member_label_en-US" xlink:label="lab_efc_ElizonTCGSBC20171Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Elizon TCG SBC 2017-1 [Member]</link:label>
    <link:label id="lab_efc_ElizonTCGSBC20171Member_documentation_en-US" xlink:label="lab_efc_ElizonTCGSBC20171Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Elizon TCG SBC 2017-1</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ElizonTCGSBC20171Member" xlink:href="efc-20201231.xsd#efc_ElizonTCGSBC20171Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ElizonTCGSBC20171Member" xlink:to="lab_efc_ElizonTCGSBC20171Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PurchaseofInvestmentsNonCash_96a25dd1-77a1-402e-af11-1366f139dbd8_negatedTerseLabel_en-US" xlink:label="lab_efc_PurchaseofInvestmentsNonCash" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchase of investments (non-cash)</link:label>
    <link:label id="lab_efc_PurchaseofInvestmentsNonCash_label_en-US" xlink:label="lab_efc_PurchaseofInvestmentsNonCash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchase of Investments Non Cash</link:label>
    <link:label id="lab_efc_PurchaseofInvestmentsNonCash_documentation_en-US" xlink:label="lab_efc_PurchaseofInvestmentsNonCash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchase of Investments Non Cash</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PurchaseofInvestmentsNonCash" xlink:href="efc-20201231.xsd#efc_PurchaseofInvestmentsNonCash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PurchaseofInvestmentsNonCash" xlink:to="lab_efc_PurchaseofInvestmentsNonCash" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_UnderlyingAssetClassAxis_135029c3-27ed-46e0-bf8d-a34a1e0c0bec_terseLabel_en-US" xlink:label="lab_us-gaap_UnderlyingAssetClassAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Underlying Asset Class [Axis]</link:label>
    <link:label id="lab_us-gaap_UnderlyingAssetClassAxis_label_en-US" xlink:label="lab_us-gaap_UnderlyingAssetClassAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Underlying Asset Class [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_UnderlyingAssetClassAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_UnderlyingAssetClassAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_UnderlyingAssetClassAxis" xlink:to="lab_us-gaap_UnderlyingAssetClassAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncomeLossFromEquityMethodInvestments_6601fd51-4bee-4e75-ba0b-8424e3d38c51_terseLabel_en-US" xlink:label="lab_us-gaap_IncomeLossFromEquityMethodInvestments" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings (losses) from investments in unconsolidated entities</link:label>
    <link:label id="lab_us-gaap_IncomeLossFromEquityMethodInvestments_97cc6202-9443-43c6-bbc3-56f46862d62c_negatedLabel_en-US" xlink:label="lab_us-gaap_IncomeLossFromEquityMethodInvestments" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">(Earnings) losses from investments in unconsolidated entities</link:label>
    <link:label id="lab_us-gaap_IncomeLossFromEquityMethodInvestments_label_en-US" xlink:label="lab_us-gaap_IncomeLossFromEquityMethodInvestments" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Income (Loss) from Equity Method Investments</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncomeLossFromEquityMethodInvestments" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncomeLossFromEquityMethodInvestments"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncomeLossFromEquityMethodInvestments" xlink:to="lab_us-gaap_IncomeLossFromEquityMethodInvestments" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Othersecuredfinancingproceeds_2fafab4e-a22d-4866-8bde-c415d4322fb4_terseLabel_en-US" xlink:label="lab_efc_Othersecuredfinancingproceeds" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other secured borrowings, Proceeds</link:label>
    <link:label id="lab_efc_Othersecuredfinancingproceeds_label_en-US" xlink:label="lab_efc_Othersecuredfinancingproceeds" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other secured financing proceeds</link:label>
    <link:label id="lab_efc_Othersecuredfinancingproceeds_documentation_en-US" xlink:label="lab_efc_Othersecuredfinancingproceeds" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds received from secured financing other than securities sold under agreements to repurchase and securities loaned.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Othersecuredfinancingproceeds" xlink:href="efc-20201231.xsd#efc_Othersecuredfinancingproceeds"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Othersecuredfinancingproceeds" xlink:to="lab_efc_Othersecuredfinancingproceeds" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PreferredStockDividendRateVariableRateSpread_0b1540f5-0e23-4c94-977b-63ebf4c20c1a_terseLabel_en-US" xlink:label="lab_efc_PreferredStockDividendRateVariableRateSpread" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock Dividend Rate Variable Rate Spread</link:label>
    <link:label id="lab_efc_PreferredStockDividendRateVariableRateSpread_label_en-US" xlink:label="lab_efc_PreferredStockDividendRateVariableRateSpread" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock Dividend Rate Variable Rate Spread</link:label>
    <link:label id="lab_efc_PreferredStockDividendRateVariableRateSpread_documentation_en-US" xlink:label="lab_efc_PreferredStockDividendRateVariableRateSpread" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The percentage rate spread over three-month LIBOR used to calculate dividend payments on preferred stock after fixed rate term ends.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PreferredStockDividendRateVariableRateSpread" xlink:href="efc-20201231.xsd#efc_PreferredStockDividendRateVariableRateSpread"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PreferredStockDividendRateVariableRateSpread" xlink:to="lab_efc_PreferredStockDividendRateVariableRateSpread" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MortgageLoansOnRealEstateCostOfMortgagesSold_8b64fb14-78e7-46ec-8dc3-350ccb7933b0_terseLabel_en-US" xlink:label="lab_us-gaap_MortgageLoansOnRealEstateCostOfMortgagesSold" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Cost of Mortgage Sold</link:label>
    <link:label id="lab_us-gaap_MortgageLoansOnRealEstateCostOfMortgagesSold_label_en-US" xlink:label="lab_us-gaap_MortgageLoansOnRealEstateCostOfMortgagesSold" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Cost of Mortgage Sold</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MortgageLoansOnRealEstateCostOfMortgagesSold" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MortgageLoansOnRealEstateCostOfMortgagesSold"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MortgageLoansOnRealEstateCostOfMortgagesSold" xlink:to="lab_us-gaap_MortgageLoansOnRealEstateCostOfMortgagesSold" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_TotalNetUnrealizedGainLossOnInvestmentsOtherSecuredBorrowingsFinancialDerivativesandForeignCurrencyTranslationexcludingFXoncash_f9b0d127-c81b-4019-a6b0-3ea5b8aee4d1_terseLabel_en-US" xlink:label="lab_efc_TotalNetUnrealizedGainLossOnInvestmentsOtherSecuredBorrowingsFinancialDerivativesandForeignCurrencyTranslationexcludingFXoncash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Net Unrealized Gain Loss On Investments, Other Secured Borrowings, Financial Derivatives, and Foreign Currency Translation (excluding FX on cash)</link:label>
    <link:label id="lab_efc_TotalNetUnrealizedGainLossOnInvestmentsOtherSecuredBorrowingsFinancialDerivativesandForeignCurrencyTranslationexcludingFXoncash_label_en-US" xlink:label="lab_efc_TotalNetUnrealizedGainLossOnInvestmentsOtherSecuredBorrowingsFinancialDerivativesandForeignCurrencyTranslationexcludingFXoncash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Net Unrealized Gain Loss On Investments, Other Secured Borrowings, Financial Derivatives, and Foreign Currency Translation (excluding FX on cash)</link:label>
    <link:label id="lab_efc_TotalNetUnrealizedGainLossOnInvestmentsOtherSecuredBorrowingsFinancialDerivativesandForeignCurrencyTranslationexcludingFXoncash_documentation_en-US" xlink:label="lab_efc_TotalNetUnrealizedGainLossOnInvestmentsOtherSecuredBorrowingsFinancialDerivativesandForeignCurrencyTranslationexcludingFXoncash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total change in net unrealized gains or losses on investments, other secured, borrowings, financial derivatives, and foreign currency translation on investments and financial derivatives. Excludes foreign currency translation on cash.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_TotalNetUnrealizedGainLossOnInvestmentsOtherSecuredBorrowingsFinancialDerivativesandForeignCurrencyTranslationexcludingFXoncash" xlink:href="efc-20201231.xsd#efc_TotalNetUnrealizedGainLossOnInvestmentsOtherSecuredBorrowingsFinancialDerivativesandForeignCurrencyTranslationexcludingFXoncash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_TotalNetUnrealizedGainLossOnInvestmentsOtherSecuredBorrowingsFinancialDerivativesandForeignCurrencyTranslationexcludingFXoncash" xlink:to="lab_efc_TotalNetUnrealizedGainLossOnInvestmentsOtherSecuredBorrowingsFinancialDerivativesandForeignCurrencyTranslationexcludingFXoncash" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Fairvalueofchargedoffloans_e9ec228b-39b8-41f3-b8de-5e7144291055_terseLabel_en-US" xlink:label="lab_efc_Fairvalueofchargedoffloans" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair value of charged-off loans</link:label>
    <link:label id="lab_efc_Fairvalueofchargedoffloans_label_en-US" xlink:label="lab_efc_Fairvalueofchargedoffloans" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair value of charged-off loans</link:label>
    <link:label id="lab_efc_Fairvalueofchargedoffloans_documentation_en-US" xlink:label="lab_efc_Fairvalueofchargedoffloans" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair value of loans that have been charged-off (loans are charged-off when they become greater than 120 days delinquent) but where it has been determined that it is probable the loan servicer will be able to collect principal and interest.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Fairvalueofchargedoffloans" xlink:href="efc-20201231.xsd#efc_Fairvalueofchargedoffloans"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Fairvalueofchargedoffloans" xlink:to="lab_efc_Fairvalueofchargedoffloans" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Maturitytermofderivativecontractunderlyingoptionscontracts_fb168ae0-a93f-4615-b563-05c99ae533ab_terseLabel_en-US" xlink:label="lab_efc_Maturitytermofderivativecontractunderlyingoptionscontracts" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity term of derivative contract underlying options contracts</link:label>
    <link:label id="lab_efc_Maturitytermofderivativecontractunderlyingoptionscontracts_label_en-US" xlink:label="lab_efc_Maturitytermofderivativecontractunderlyingoptionscontracts" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity term of derivative contract underlying options contracts</link:label>
    <link:label id="lab_efc_Maturitytermofderivativecontractunderlyingoptionscontracts_documentation_en-US" xlink:label="lab_efc_Maturitytermofderivativecontractunderlyingoptionscontracts" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity term of derivative contract underlying options contracts</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Maturitytermofderivativecontractunderlyingoptionscontracts" xlink:href="efc-20201231.xsd#efc_Maturitytermofderivativecontractunderlyingoptionscontracts"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Maturitytermofderivativecontractunderlyingoptionscontracts" xlink:to="lab_efc_Maturitytermofderivativecontractunderlyingoptionscontracts" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CorporateDebtSoldShortMember_cf4685d9-7fa7-43e4-86f1-d0ffccff06af_terseLabel_en-US" xlink:label="lab_efc_CorporateDebtSoldShortMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate debt securities</link:label>
    <link:label id="lab_efc_CorporateDebtSoldShortMember_label_en-US" xlink:label="lab_efc_CorporateDebtSoldShortMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate Debt Sold Short [Member]</link:label>
    <link:label id="lab_efc_CorporateDebtSoldShortMember_documentation_en-US" xlink:label="lab_efc_CorporateDebtSoldShortMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate Debt Sold Short [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CorporateDebtSoldShortMember" xlink:href="efc-20201231.xsd#efc_CorporateDebtSoldShortMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CorporateDebtSoldShortMember" xlink:to="lab_efc_CorporateDebtSoldShortMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentCompanyExpenseRatioAfterIncentiveAllocation_ae46433a-f837-472b-bf84-38a28e29b135_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_InvestmentCompanyExpenseRatioAfterIncentiveAllocation" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Expense Ratio after Incentive Allocation</link:label>
    <link:label id="lab_us-gaap_InvestmentCompanyExpenseRatioAfterIncentiveAllocation_label_en-US" xlink:label="lab_us-gaap_InvestmentCompanyExpenseRatioAfterIncentiveAllocation" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Expense Ratio after Incentive Allocation</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentCompanyExpenseRatioAfterIncentiveAllocation" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentCompanyExpenseRatioAfterIncentiveAllocation"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentCompanyExpenseRatioAfterIncentiveAllocation" xlink:to="lab_us-gaap_InvestmentCompanyExpenseRatioAfterIncentiveAllocation" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Economicinterestretained_b34f065c-5e51-4c66-b065-aa7d9a1ac05b_terseLabel_en-US" xlink:label="lab_efc_Economicinterestretained" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Economic interest retained</link:label>
    <link:label id="lab_efc_Economicinterestretained_label_en-US" xlink:label="lab_efc_Economicinterestretained" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Economic interest retained</link:label>
    <link:label id="lab_efc_Economicinterestretained_documentation_en-US" xlink:label="lab_efc_Economicinterestretained" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Economic interest in CLO securitization retained by affiliated entity in compliance with risk retention rules.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Economicinterestretained" xlink:href="efc-20201231.xsd#efc_Economicinterestretained"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Economicinterestretained" xlink:to="lab_efc_Economicinterestretained" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquityMethodInvestmentsAndJointVenturesAbstract_88642d14-2249-4407-8051-7632ea9066db_terseLabel_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentsAndJointVenturesAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investments and Joint Ventures [Abstract]</link:label>
    <link:label id="lab_us-gaap_EquityMethodInvestmentsAndJointVenturesAbstract_label_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentsAndJointVenturesAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investments and Joint Ventures [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquityMethodInvestmentsAndJointVenturesAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquityMethodInvestmentsAndJointVenturesAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquityMethodInvestmentsAndJointVenturesAbstract" xlink:to="lab_us-gaap_EquityMethodInvestmentsAndJointVenturesAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FinancingReceivableNumberOfLoans_be131dff-f5e3-454d-8b13-594e7ebc8d02_terseLabel_en-US" xlink:label="lab_efc_FinancingReceivableNumberOfLoans" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of loans</link:label>
    <link:label id="lab_efc_FinancingReceivableNumberOfLoans_label_en-US" xlink:label="lab_efc_FinancingReceivableNumberOfLoans" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable, Number Of Loans</link:label>
    <link:label id="lab_efc_FinancingReceivableNumberOfLoans_documentation_en-US" xlink:label="lab_efc_FinancingReceivableNumberOfLoans" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable, Number Of Loans</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FinancingReceivableNumberOfLoans" xlink:href="efc-20201231.xsd#efc_FinancingReceivableNumberOfLoans"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FinancingReceivableNumberOfLoans" xlink:to="lab_efc_FinancingReceivableNumberOfLoans" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsOtherFourPointFivePercentMember_5d27bdc1-c13f-4f80-a084-1bfbbd5dc7d5_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsOtherFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Other) Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsOtherFourPointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsOtherFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Other) Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsOtherFourPointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsOtherFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Other) Four Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationPoolsOtherFourPointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationPoolsOtherFourPointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationPoolsOtherFourPointFivePercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationPoolsOtherFourPointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StockIssuedDuringPeriodSharesConversionOfUnits_5f200aff-5755-4a54-b237-5d1236b2a95d_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_StockIssuedDuringPeriodSharesConversionOfUnits" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stock issued during period, shares, conversion of units</link:label>
    <link:label id="lab_us-gaap_StockIssuedDuringPeriodSharesConversionOfUnits_53bfb365-708d-45bf-a6bb-cc6f4e7cf0a5_terseLabel_en-US" xlink:label="lab_us-gaap_StockIssuedDuringPeriodSharesConversionOfUnits" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stock issued during period, shares, conversion of units</link:label>
    <link:label id="lab_us-gaap_StockIssuedDuringPeriodSharesConversionOfUnits_label_en-US" xlink:label="lab_us-gaap_StockIssuedDuringPeriodSharesConversionOfUnits" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stock Issued During Period, Shares, Conversion of Units</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StockIssuedDuringPeriodSharesConversionOfUnits" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockIssuedDuringPeriodSharesConversionOfUnits"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StockIssuedDuringPeriodSharesConversionOfUnits" xlink:to="lab_us-gaap_StockIssuedDuringPeriodSharesConversionOfUnits" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentsMember_1a426bef-9f02-49b5-b7d6-d3b89abf796c_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments</link:label>
    <link:label id="lab_us-gaap_InvestmentsMember_label_en-US" xlink:label="lab_us-gaap_InvestmentsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentsMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentsMember" xlink:to="lab_us-gaap_InvestmentsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritiespurchasedunderagreementstoresellDomain_9f10f950-15d4-4cdf-b85a-579a7581b37c_terseLabel_en-US" xlink:label="lab_efc_SecuritiespurchasedunderagreementstoresellDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities purchased under agreements to resell [Domain]</link:label>
    <link:label id="lab_efc_SecuritiespurchasedunderagreementstoresellDomain_label_en-US" xlink:label="lab_efc_SecuritiespurchasedunderagreementstoresellDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities purchased under agreements to resell [Domain]</link:label>
    <link:label id="lab_efc_SecuritiespurchasedunderagreementstoresellDomain_documentation_en-US" xlink:label="lab_efc_SecuritiespurchasedunderagreementstoresellDomain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Domain] for Securities purchased under agreements to resell [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritiespurchasedunderagreementstoresellDomain" xlink:href="efc-20201231.xsd#efc_SecuritiespurchasedunderagreementstoresellDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritiespurchasedunderagreementstoresellDomain" xlink:to="lab_efc_SecuritiespurchasedunderagreementstoresellDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ParticipationinmultiborrowerfinancingfacilityMember_5ff5c438-7ff6-4c43-8a46-bcb0962717f9_verboseLabel_en-US" xlink:label="lab_efc_ParticipationinmultiborrowerfinancingfacilityMember" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Participation in multi-borrower financing facility</link:label>
    <link:label id="lab_efc_ParticipationinmultiborrowerfinancingfacilityMember_label_en-US" xlink:label="lab_efc_ParticipationinmultiborrowerfinancingfacilityMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Participation in multi-borrower financing facility [Member]</link:label>
    <link:label id="lab_efc_ParticipationinmultiborrowerfinancingfacilityMember_documentation_en-US" xlink:label="lab_efc_ParticipationinmultiborrowerfinancingfacilityMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Participation in multi-borrower financing facility [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ParticipationinmultiborrowerfinancingfacilityMember" xlink:href="efc-20201231.xsd#efc_ParticipationinmultiborrowerfinancingfacilityMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ParticipationinmultiborrowerfinancingfacilityMember" xlink:to="lab_efc_ParticipationinmultiborrowerfinancingfacilityMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MinorityInterestDisclosureTextBlock_0ba6956a-40b1-4d77-a088-b9a95dbc9c99_terseLabel_en-US" xlink:label="lab_us-gaap_MinorityInterestDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling Interests</link:label>
    <link:label id="lab_us-gaap_MinorityInterestDisclosureTextBlock_label_en-US" xlink:label="lab_us-gaap_MinorityInterestDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noncontrolling Interest Disclosure [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MinorityInterestDisclosureTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MinorityInterestDisclosureTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MinorityInterestDisclosureTextBlock" xlink:to="lab_us-gaap_MinorityInterestDisclosureTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquitySecuritiesMember_f50327c4-e738-4460-93bc-885ab9b0e0e8_terseLabel_en-US" xlink:label="lab_us-gaap_EquitySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate equity securities</link:label>
    <link:label id="lab_us-gaap_EquitySecuritiesMember_label_en-US" xlink:label="lab_us-gaap_EquitySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquitySecuritiesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquitySecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquitySecuritiesMember" xlink:to="lab_us-gaap_EquitySecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Repaymentsofreverserepurchaseagreementsnoncash_b4eb140b-5702-46dc-b112-6e2801299f1b_terseLabel_en-US" xlink:label="lab_efc_Repaymentsofreverserepurchaseagreementsnoncash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repayments of reverse repurchase agreements non cash</link:label>
    <link:label id="lab_efc_Repaymentsofreverserepurchaseagreementsnoncash_label_en-US" xlink:label="lab_efc_Repaymentsofreverserepurchaseagreementsnoncash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repayments of reverse repurchase agreements non cash</link:label>
    <link:label id="lab_efc_Repaymentsofreverserepurchaseagreementsnoncash_documentation_en-US" xlink:label="lab_efc_Repaymentsofreverserepurchaseagreementsnoncash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repayments of reverse repurchase agreements non cash</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Repaymentsofreverserepurchaseagreementsnoncash" xlink:href="efc-20201231.xsd#efc_Repaymentsofreverserepurchaseagreementsnoncash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Repaymentsofreverserepurchaseagreementsnoncash" xlink:to="lab_efc_Repaymentsofreverserepurchaseagreementsnoncash" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodGross_fb7f3e90-e252-4689-953a-c3453a447254_verboseLabel_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodGross" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Granted</link:label>
    <link:label id="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodGross_label_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodGross" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Compensation Arrangement by Share-based Payment Award, Options, Grants in Period, Gross</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodGross" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodGross"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodGross" xlink:to="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsGrantsInPeriodGross" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember_d32e7637-1064-4d61-8142-3e8884657674_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Four Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFourPointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_UnrelatedthirdpartyjointventureinterestMember_70957012-5120-429c-8b98-2da426174964_terseLabel_en-US" xlink:label="lab_efc_UnrelatedthirdpartyjointventureinterestMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrelated third party joint venture interest</link:label>
    <link:label id="lab_efc_UnrelatedthirdpartyjointventureinterestMember_label_en-US" xlink:label="lab_efc_UnrelatedthirdpartyjointventureinterestMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrelated third party joint venture interest [Member]</link:label>
    <link:label id="lab_efc_UnrelatedthirdpartyjointventureinterestMember_documentation_en-US" xlink:label="lab_efc_UnrelatedthirdpartyjointventureinterestMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrelated third party joint venture interest [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_UnrelatedthirdpartyjointventureinterestMember" xlink:href="efc-20201231.xsd#efc_UnrelatedthirdpartyjointventureinterestMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_UnrelatedthirdpartyjointventureinterestMember" xlink:to="lab_efc_UnrelatedthirdpartyjointventureinterestMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LongtermDebtMaturitiesRepaymentsofPrincipalTotal_6d20a0c9-3265-4cdf-8342-1d32a2d4fce3_totalLabel_en-US" xlink:label="lab_efc_LongtermDebtMaturitiesRepaymentsofPrincipalTotal" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Scheduled Repayment of Principal</link:label>
    <link:label id="lab_efc_LongtermDebtMaturitiesRepaymentsofPrincipalTotal_label_en-US" xlink:label="lab_efc_LongtermDebtMaturitiesRepaymentsofPrincipalTotal" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Maturities, Repayments of Principal, Total</link:label>
    <link:label id="lab_efc_LongtermDebtMaturitiesRepaymentsofPrincipalTotal_documentation_en-US" xlink:label="lab_efc_LongtermDebtMaturitiesRepaymentsofPrincipalTotal" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total amount of long-term debt payable, sinking fund requirements, and other securities issued that are redeemable by holder at fixed or determinable prices and dates maturing.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LongtermDebtMaturitiesRepaymentsofPrincipalTotal" xlink:href="efc-20201231.xsd#efc_LongtermDebtMaturitiesRepaymentsofPrincipalTotal"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LongtermDebtMaturitiesRepaymentsofPrincipalTotal" xlink:to="lab_efc_LongtermDebtMaturitiesRepaymentsofPrincipalTotal" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearSixPercentMember_cc2e0bf7-48de-4915-994c-f4825e5f6f61_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearSixPercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Six Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearSixPercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearSixPercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Six Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearSixPercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearSixPercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Six Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearSixPercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearSixPercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearSixPercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearSixPercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityCurrentReportingStatus_dc30b7a3-b63d-411a-b768-f5917daba7c5_terseLabel_en-US" xlink:label="lab_dei_EntityCurrentReportingStatus" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Current Reporting Status</link:label>
    <link:label id="lab_dei_EntityCurrentReportingStatus_label_en-US" xlink:label="lab_dei_EntityCurrentReportingStatus" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Current Reporting Status</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityCurrentReportingStatus" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityCurrentReportingStatus"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityCurrentReportingStatus" xlink:to="lab_dei_EntityCurrentReportingStatus" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_USTreasurySecuritiesMember_99fe7557-4514-4ffc-a72e-888c33795105_terseLabel_en-US" xlink:label="lab_us-gaap_USTreasurySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury securities</link:label>
    <link:label id="lab_us-gaap_USTreasurySecuritiesMember_label_en-US" xlink:label="lab_us-gaap_USTreasurySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">US Treasury Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_USTreasurySecuritiesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_USTreasurySecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_USTreasurySecuritiesMember" xlink:to="lab_us-gaap_USTreasurySecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfAssetsSoldUnderAgreementsToRepurchaseTable_6e82ed4e-2cb1-47ab-8e08-04c99ba136f2_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfAssetsSoldUnderAgreementsToRepurchaseTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Assets Sold under Agreements to Repurchase [Table]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfAssetsSoldUnderAgreementsToRepurchaseTable_label_en-US" xlink:label="lab_us-gaap_ScheduleOfAssetsSoldUnderAgreementsToRepurchaseTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Assets Sold under Agreements to Repurchase [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfAssetsSoldUnderAgreementsToRepurchaseTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfAssetsSoldUnderAgreementsToRepurchaseTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfAssetsSoldUnderAgreementsToRepurchaseTable" xlink:to="lab_us-gaap_ScheduleOfAssetsSoldUnderAgreementsToRepurchaseTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsMember_a72dd6d6-c3e7-4d9a-902d-7139d211c40a_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationPoolsMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationPoolsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationPoolsMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationPoolsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NoncontrollingInterestAbstract_8bd5a940-2236-4381-bc0c-77f9814ede53_terseLabel_en-US" xlink:label="lab_us-gaap_NoncontrollingInterestAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noncontrolling Interest [Abstract]</link:label>
    <link:label id="lab_us-gaap_NoncontrollingInterestAbstract_label_en-US" xlink:label="lab_us-gaap_NoncontrollingInterestAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noncontrolling Interest [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NoncontrollingInterestAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NoncontrollingInterestAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NoncontrollingInterestAbstract" xlink:to="lab_us-gaap_NoncontrollingInterestAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Distributionsfrominvestmentsinunconsolidatedentities_9be23018-8297-4c80-8836-51621a4b3de8_terseLabel_en-US" xlink:label="lab_efc_Distributionsfrominvestmentsinunconsolidatedentities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Distributions from investments in unconsolidated entities</link:label>
    <link:label id="lab_efc_Distributionsfrominvestmentsinunconsolidatedentities_label_en-US" xlink:label="lab_efc_Distributionsfrominvestmentsinunconsolidatedentities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Distributions from investments in unconsolidated entities</link:label>
    <link:label id="lab_efc_Distributionsfrominvestmentsinunconsolidatedentities_documentation_en-US" xlink:label="lab_efc_Distributionsfrominvestmentsinunconsolidatedentities" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Distributions from investments in unconsolidated entities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Distributionsfrominvestmentsinunconsolidatedentities" xlink:href="efc-20201231.xsd#efc_Distributionsfrominvestmentsinunconsolidatedentities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Distributionsfrominvestmentsinunconsolidatedentities" xlink:to="lab_efc_Distributionsfrominvestmentsinunconsolidatedentities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EstimateweightedaveragelifeAxis_eb95488b-eea2-462f-a94c-70cb1906b120_terseLabel_en-US" xlink:label="lab_efc_EstimateweightedaveragelifeAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Estimate weighted average life [Axis]</link:label>
    <link:label id="lab_efc_EstimateweightedaveragelifeAxis_label_en-US" xlink:label="lab_efc_EstimateweightedaveragelifeAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Estimate weighted average life [Axis]</link:label>
    <link:label id="lab_efc_EstimateweightedaveragelifeAxis_documentation_en-US" xlink:label="lab_efc_EstimateweightedaveragelifeAxis" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Estimate weighted average life [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EstimateweightedaveragelifeAxis" xlink:href="efc-20201231.xsd#efc_EstimateweightedaveragelifeAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EstimateweightedaveragelifeAxis" xlink:to="lab_efc_EstimateweightedaveragelifeAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ForwardContractsMember_287dac00-a37b-4ff8-9a7f-2c647e22fdde_terseLabel_en-US" xlink:label="lab_us-gaap_ForwardContractsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Forwards</link:label>
    <link:label id="lab_us-gaap_ForwardContractsMember_label_en-US" xlink:label="lab_us-gaap_ForwardContractsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Forward Contracts [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ForwardContractsMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ForwardContractsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ForwardContractsMember" xlink:to="lab_us-gaap_ForwardContractsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CommonStockMember_0dc1a828-d921-4cf2-8a75-1235a5fdca15_terseLabel_en-US" xlink:label="lab_us-gaap_CommonStockMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common Stock</link:label>
    <link:label id="lab_us-gaap_CommonStockMember_label_en-US" xlink:label="lab_us-gaap_CommonStockMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common Stock [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CommonStockMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CommonStockMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CommonStockMember" xlink:to="lab_us-gaap_CommonStockMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InterestRateCurrencyRiskMember_9224a8f5-cd47-42f8-92b3-5b84e080827f_terseLabel_en-US" xlink:label="lab_efc_InterestRateCurrencyRiskMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Rate/Currency Risk [Member]</link:label>
    <link:label id="lab_efc_InterestRateCurrencyRiskMember_label_en-US" xlink:label="lab_efc_InterestRateCurrencyRiskMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Rate/Currency Risk [Member]</link:label>
    <link:label id="lab_efc_InterestRateCurrencyRiskMember_documentation_en-US" xlink:label="lab_efc_InterestRateCurrencyRiskMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Rate/Currency Risk [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InterestRateCurrencyRiskMember" xlink:href="efc-20201231.xsd#efc_InterestRateCurrencyRiskMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InterestRateCurrencyRiskMember" xlink:to="lab_efc_InterestRateCurrencyRiskMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationSixPercentMember_d95fd56b-4128-4df7-b368-617d36f3649e_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationSixPercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Six Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationSixPercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationSixPercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Six Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationSixPercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationSixPercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Six Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationSixPercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationSixPercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationSixPercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationSixPercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember_daf57805-89a8-4740-88f4-cb3856599b2f_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Five Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Five Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Five Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_UnrealizedGainLossOnDerivativesAndCommodityContractsAbstract_6d1bc39c-fcf0-40e8-8fb5-875194baf528_terseLabel_en-US" xlink:label="lab_us-gaap_UnrealizedGainLossOnDerivativesAndCommodityContractsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized Gain (Loss) on Derivatives and Commodity Contracts [Abstract]</link:label>
    <link:label id="lab_us-gaap_UnrealizedGainLossOnDerivativesAndCommodityContractsAbstract_label_en-US" xlink:label="lab_us-gaap_UnrealizedGainLossOnDerivativesAndCommodityContractsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized Gain (Loss) on Derivatives and Commodity Contracts [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_UnrealizedGainLossOnDerivativesAndCommodityContractsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_UnrealizedGainLossOnDerivativesAndCommodityContractsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_UnrealizedGainLossOnDerivativesAndCommodityContractsAbstract" xlink:to="lab_us-gaap_UnrealizedGainLossOnDerivativesAndCommodityContractsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EarningsPerShareDilutedUndistributed_bfe28b9d-23b4-4b97-accc-6ecbedb3e479_terseLabel_en-US" xlink:label="lab_us-gaap_EarningsPerShareDilutedUndistributed" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Undistributed (Distributed in excess of) (usd per share)</link:label>
    <link:label id="lab_us-gaap_EarningsPerShareDilutedUndistributed_label_en-US" xlink:label="lab_us-gaap_EarningsPerShareDilutedUndistributed" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share, Diluted, Undistributed</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EarningsPerShareDilutedUndistributed" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EarningsPerShareDilutedUndistributed"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EarningsPerShareDilutedUndistributed" xlink:to="lab_us-gaap_EarningsPerShareDilutedUndistributed" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NetIncomeLossAttributabletoNonParticipatingNoncontrollingInterest_93cc3e50-5c34-4724-a17d-9070fa64de2d_terseLabel_en-US" xlink:label="lab_efc_NetIncomeLossAttributabletoNonParticipatingNoncontrollingInterest" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net increase (decrease) in equity resulting from operations</link:label>
    <link:label id="lab_efc_NetIncomeLossAttributabletoNonParticipatingNoncontrollingInterest_label_en-US" xlink:label="lab_efc_NetIncomeLossAttributabletoNonParticipatingNoncontrollingInterest" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Income (Loss) Attributable to Non-Participating Non-controlling Interest</link:label>
    <link:label id="lab_efc_NetIncomeLossAttributabletoNonParticipatingNoncontrollingInterest_documentation_en-US" xlink:label="lab_efc_NetIncomeLossAttributabletoNonParticipatingNoncontrollingInterest" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net income (loss) attributable to non-controlling interest that does not own units convertible to common shares. This income (loss) is excluded in the calculation of earnings per share.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NetIncomeLossAttributabletoNonParticipatingNoncontrollingInterest" xlink:href="efc-20201231.xsd#efc_NetIncomeLossAttributabletoNonParticipatingNoncontrollingInterest"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NetIncomeLossAttributabletoNonParticipatingNoncontrollingInterest" xlink:to="lab_efc_NetIncomeLossAttributabletoNonParticipatingNoncontrollingInterest" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CreditPortfolioMember_4b187904-3803-46df-b8f4-14c9ca841987_terseLabel_en-US" xlink:label="lab_efc_CreditPortfolioMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit</link:label>
    <link:label id="lab_efc_CreditPortfolioMember_label_en-US" xlink:label="lab_efc_CreditPortfolioMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Portfolio [Member]</link:label>
    <link:label id="lab_efc_CreditPortfolioMember_documentation_en-US" xlink:label="lab_efc_CreditPortfolioMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments in the Company's credit portfolio. Such investments can include non-Agency (private issued) RMBS and CMBS, collateralized loan obligations, and consumer loans.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CreditPortfolioMember" xlink:href="efc-20201231.xsd#efc_CreditPortfolioMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CreditPortfolioMember" xlink:to="lab_efc_CreditPortfolioMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_TotalReturnSwapMember_7f1d38e3-0ad6-4995-a406-66b4f4f8af6f_terseLabel_en-US" xlink:label="lab_us-gaap_TotalReturnSwapMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total return swaps</link:label>
    <link:label id="lab_us-gaap_TotalReturnSwapMember_label_en-US" xlink:label="lab_us-gaap_TotalReturnSwapMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Return Swap [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_TotalReturnSwapMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_TotalReturnSwapMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_TotalReturnSwapMember" xlink:to="lab_us-gaap_TotalReturnSwapMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardLineItems_f3699127-fb25-4289-828e-321220f262ac_terseLabel_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Compensation Arrangement by Share-based Payment Award [Line Items]</link:label>
    <link:label id="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardLineItems_label_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Compensation Arrangement by Share-based Payment Award [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardLineItems" xlink:to="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProceedsFromIssuanceOfPreferredStockAndPreferenceStock_d0f76541-6a5c-49ee-a61f-4adf774bf129_terseLabel_en-US" xlink:label="lab_us-gaap_ProceedsFromIssuanceOfPreferredStockAndPreferenceStock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from Issuance of Preferred Stock and Preference Stock</link:label>
    <link:label id="lab_us-gaap_ProceedsFromIssuanceOfPreferredStockAndPreferenceStock_label_en-US" xlink:label="lab_us-gaap_ProceedsFromIssuanceOfPreferredStockAndPreferenceStock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from Issuance of Preferred Stock and Preference Stock</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProceedsFromIssuanceOfPreferredStockAndPreferenceStock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromIssuanceOfPreferredStockAndPreferenceStock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProceedsFromIssuanceOfPreferredStockAndPreferenceStock" xlink:to="lab_us-gaap_ProceedsFromIssuanceOfPreferredStockAndPreferenceStock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202310MemberDomain_0fe83abc-4176-41d9-a44b-54e428467e12_terseLabel_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202310MemberDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.88 Percent, Maturity Date 2023 10 [Member] [Domain]</link:label>
    <link:label id="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202310MemberDomain_label_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202310MemberDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.88 Percent, Maturity Date 2023 10 [Member] [Domain]</link:label>
    <link:label id="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202310MemberDomain_documentation_en-US" xlink:label="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202310MemberDomain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 2.88 Percent, Maturity Date 2023 10 [Member] [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_U.S.TreasuryNote2.88PercentMaturityDate202310MemberDomain" xlink:href="efc-20201231.xsd#efc_U.S.TreasuryNote2.88PercentMaturityDate202310MemberDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_U.S.TreasuryNote2.88PercentMaturityDate202310MemberDomain" xlink:to="lab_efc_U.S.TreasuryNote2.88PercentMaturityDate202310MemberDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember_aeb865d6-a9cc-47a0-b416-3bf9ceeb96d6_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Five Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Five Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Five Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearFivePointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_IL_39da237e-6d24-4458-9d6a-7d78298f40e5_terseLabel_en-US" xlink:label="lab_stpr_IL" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">ILLINOIS</link:label>
    <link:label id="lab_stpr_IL_label_en-US" xlink:label="lab_stpr_IL" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">ILLINOIS</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_IL" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_IL"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_IL" xlink:to="lab_stpr_IL" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProfitLoss_98d1fc7b-c4a9-41e2-97f5-a49a12a93eeb_totalLabel_en-US" xlink:label="lab_us-gaap_ProfitLoss" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Income (Loss), Including Portion Attributable to Noncontrolling Interest, Total</link:label>
    <link:label id="lab_us-gaap_ProfitLoss_13ddbdec-2010-45c1-8acd-13ebf1985374_terseLabel_en-US" xlink:label="lab_us-gaap_ProfitLoss" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Income (Loss), Including Portion Attributable to Noncontrolling Interest</link:label>
    <link:label id="lab_us-gaap_ProfitLoss_label_en-US" xlink:label="lab_us-gaap_ProfitLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Income (Loss), Including Portion Attributable to Noncontrolling Interest</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProfitLoss" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProfitLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProfitLoss" xlink:to="lab_us-gaap_ProfitLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleofCreditDefaultSwapsTableTextBlock_92fc509b-8eb4-4d20-8ccc-741b61a01fa2_terseLabel_en-US" xlink:label="lab_efc_ScheduleofCreditDefaultSwapsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Credit Default Swaps [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleofCreditDefaultSwapsTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleofCreditDefaultSwapsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Credit Default Swaps [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleofCreditDefaultSwapsTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleofCreditDefaultSwapsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Tabular disclosure of credit default swaps.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleofCreditDefaultSwapsTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleofCreditDefaultSwapsTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleofCreditDefaultSwapsTableTextBlock" xlink:to="lab_efc_ScheduleofCreditDefaultSwapsTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_AffiliatedEntityMember_7cf0dc99-10ab-4714-bcfd-1078358613b0_terseLabel_en-US" xlink:label="lab_srt_AffiliatedEntityMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Affiliated Entity [Member]</link:label>
    <link:label id="lab_srt_AffiliatedEntityMember_label_en-US" xlink:label="lab_srt_AffiliatedEntityMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Affiliated Entity [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_AffiliatedEntityMember" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_AffiliatedEntityMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_AffiliatedEntityMember" xlink:to="lab_srt_AffiliatedEntityMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OtherInvestmentSecuritiesMember_9c8ccc2c-e54d-43a9-b4c7-75e0669d0d42_terseLabel_en-US" xlink:label="lab_efc_OtherInvestmentSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Securities</link:label>
    <link:label id="lab_efc_OtherInvestmentSecuritiesMember_label_en-US" xlink:label="lab_efc_OtherInvestmentSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Investment Securities [Member]</link:label>
    <link:label id="lab_efc_OtherInvestmentSecuritiesMember_documentation_en-US" xlink:label="lab_efc_OtherInvestmentSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Investment Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OtherInvestmentSecuritiesMember" xlink:href="efc-20201231.xsd#efc_OtherInvestmentSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OtherInvestmentSecuritiesMember" xlink:to="lab_efc_OtherInvestmentSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntitySmallBusiness_c0025872-19b4-4397-ae99-9ed42de03ad1_terseLabel_en-US" xlink:label="lab_dei_EntitySmallBusiness" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Small Business</link:label>
    <link:label id="lab_dei_EntitySmallBusiness_label_en-US" xlink:label="lab_dei_EntitySmallBusiness" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Small Business</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntitySmallBusiness" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntitySmallBusiness"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntitySmallBusiness" xlink:to="lab_dei_EntitySmallBusiness" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_IN_685eb67b-ac6e-4fdf-b35e-fcfeafdb2b71_terseLabel_en-US" xlink:label="lab_stpr_IN" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Indiana [Member]</link:label>
    <link:label id="lab_stpr_IN_label_en-US" xlink:label="lab_stpr_IN" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">INDIANA</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_IN" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_IN"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_IN" xlink:to="lab_stpr_IN" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FinancialMember_e2e76bcf-7bff-4f1a-8e39-e70696527177_terseLabel_en-US" xlink:label="lab_efc_FinancialMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial [Member]</link:label>
    <link:label id="lab_efc_FinancialMember_label_en-US" xlink:label="lab_efc_FinancialMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial [Member]</link:label>
    <link:label id="lab_efc_FinancialMember_documentation_en-US" xlink:label="lab_efc_FinancialMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FinancialMember" xlink:href="efc-20201231.xsd#efc_FinancialMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FinancialMember" xlink:to="lab_efc_FinancialMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PaymentsToAcquireInvestments_584a5bf5-decd-4661-9b5c-5c14ca5b20f0_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_PaymentsToAcquireInvestments" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchase of investments</link:label>
    <link:label id="lab_us-gaap_PaymentsToAcquireInvestments_label_en-US" xlink:label="lab_us-gaap_PaymentsToAcquireInvestments" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments to Acquire Investments</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PaymentsToAcquireInvestments" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsToAcquireInvestments"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PaymentsToAcquireInvestments" xlink:to="lab_us-gaap_PaymentsToAcquireInvestments" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersIntoLevel3_2df6e919-9ef4-4f1e-8f72-0f8db86ece3b_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersIntoLevel3" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets, Transfers into Level 3</link:label>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersIntoLevel3_label_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersIntoLevel3" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement with Unobservable Inputs Reconciliation, Recurring Basis, Asset Transfers Into Level 3</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersIntoLevel3" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersIntoLevel3"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersIntoLevel3" xlink:to="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetTransfersIntoLevel3" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityFilerCategory_a76e0c9c-6669-49fa-9b60-386b2c78174b_terseLabel_en-US" xlink:label="lab_dei_EntityFilerCategory" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Filer Category</link:label>
    <link:label id="lab_dei_EntityFilerCategory_label_en-US" xlink:label="lab_dei_EntityFilerCategory" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Filer Category</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityFilerCategory" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityFilerCategory"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityFilerCategory" xlink:to="lab_dei_EntityFilerCategory" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearFour_488a6fee-729f-4411-8fe0-ebcdd9d060db_terseLabel_en-US" xlink:label="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearFour" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2021 Scheduled Repayment of Principal</link:label>
    <link:label id="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearFour_label_en-US" xlink:label="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearFour" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Maturities, Repayments of Principal in Rolling Year Four</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearFour" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearFour"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearFour" xlink:to="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInRollingYearFour" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProceedsFromSaleOfAvailableForSaleSecurities_16c04716-4a7d-43e2-9fe9-9e7fb44de9d0_terseLabel_en-US" xlink:label="lab_us-gaap_ProceedsFromSaleOfAvailableForSaleSecurities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds</link:label>
    <link:label id="lab_us-gaap_ProceedsFromSaleOfAvailableForSaleSecurities_label_en-US" xlink:label="lab_us-gaap_ProceedsFromSaleOfAvailableForSaleSecurities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from Sale of Available-for-sale Securities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProceedsFromSaleOfAvailableForSaleSecurities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromSaleOfAvailableForSaleSecurities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProceedsFromSaleOfAvailableForSaleSecurities" xlink:to="lab_us-gaap_ProceedsFromSaleOfAvailableForSaleSecurities" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DeferredIncomeTaxLiabilities_3541d590-afbf-4cbd-8a13-35c2360fe65c_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_DeferredIncomeTaxLiabilities" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Tax Liabilities, Gross</link:label>
    <link:label id="lab_us-gaap_DeferredIncomeTaxLiabilities_label_en-US" xlink:label="lab_us-gaap_DeferredIncomeTaxLiabilities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Tax Liabilities, Gross</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DeferredIncomeTaxLiabilities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DeferredIncomeTaxLiabilities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DeferredIncomeTaxLiabilities" xlink:to="lab_us-gaap_DeferredIncomeTaxLiabilities" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueOfFinancialInstrumentsPolicy_0d37ccc2-4f68-4797-820d-340b79d18ae7_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueOfFinancialInstrumentsPolicy" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Valuation</link:label>
    <link:label id="lab_us-gaap_FairValueOfFinancialInstrumentsPolicy_label_en-US" xlink:label="lab_us-gaap_FairValueOfFinancialInstrumentsPolicy" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value of Financial Instruments, Policy [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueOfFinancialInstrumentsPolicy" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueOfFinancialInstrumentsPolicy"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueOfFinancialInstrumentsPolicy" xlink:to="lab_us-gaap_FairValueOfFinancialInstrumentsPolicy" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentCompanyFinancialHighlightsTextBlock_d03949c5-0c4e-4a19-b77a-6aa5e7515492_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentCompanyFinancialHighlightsTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Financial Highlights</link:label>
    <link:label id="lab_us-gaap_InvestmentCompanyFinancialHighlightsTextBlock_label_en-US" xlink:label="lab_us-gaap_InvestmentCompanyFinancialHighlightsTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Financial Highlights [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentCompanyFinancialHighlightsTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentCompanyFinancialHighlightsTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentCompanyFinancialHighlightsTextBlock" xlink:to="lab_us-gaap_InvestmentCompanyFinancialHighlightsTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InterestRateSwapMember_3955272b-fec5-4967-aa6f-f671af02b07b_terseLabel_en-US" xlink:label="lab_us-gaap_InterestRateSwapMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest rate swaps</link:label>
    <link:label id="lab_us-gaap_InterestRateSwapMember_label_en-US" xlink:label="lab_us-gaap_InterestRateSwapMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Rate Swap [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InterestRateSwapMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InterestRateSwapMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InterestRateSwapMember" xlink:to="lab_us-gaap_InterestRateSwapMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RBCCapitalMarketsLLCCollateralizedbyParValue500ExchangeTradedCorporateDebtCoupon5.75MaturityDate202210Member_82db5772-d2f3-42ce-a3f2-16a9dbe6a0ae_terseLabel_en-US" xlink:label="lab_efc_RBCCapitalMarketsLLCCollateralizedbyParValue500ExchangeTradedCorporateDebtCoupon5.75MaturityDate202210Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">RBC Capital Markets LLC Collateralized by Par Value $500 Exchange-Traded Corporate Debt, Coupon 5.75%, Maturity Date 2022 10 [Member]</link:label>
    <link:label id="lab_efc_RBCCapitalMarketsLLCCollateralizedbyParValue500ExchangeTradedCorporateDebtCoupon5.75MaturityDate202210Member_label_en-US" xlink:label="lab_efc_RBCCapitalMarketsLLCCollateralizedbyParValue500ExchangeTradedCorporateDebtCoupon5.75MaturityDate202210Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">RBC Capital Markets LLC Collateralized by Par Value $500 Exchange-Traded Corporate Debt, Coupon 5.75%, Maturity Date 2022 10 [Member]</link:label>
    <link:label id="lab_efc_RBCCapitalMarketsLLCCollateralizedbyParValue500ExchangeTradedCorporateDebtCoupon5.75MaturityDate202210Member_documentation_en-US" xlink:label="lab_efc_RBCCapitalMarketsLLCCollateralizedbyParValue500ExchangeTradedCorporateDebtCoupon5.75MaturityDate202210Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">RBC Capital Markets LLC Collateralized by Par Value $500 Exchange-Traded Corporate Debt, Coupon 5.75%, Maturity Date 2022 10 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RBCCapitalMarketsLLCCollateralizedbyParValue500ExchangeTradedCorporateDebtCoupon5.75MaturityDate202210Member" xlink:href="efc-20201231.xsd#efc_RBCCapitalMarketsLLCCollateralizedbyParValue500ExchangeTradedCorporateDebtCoupon5.75MaturityDate202210Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RBCCapitalMarketsLLCCollateralizedbyParValue500ExchangeTradedCorporateDebtCoupon5.75MaturityDate202210Member" xlink:to="lab_efc_RBCCapitalMarketsLLCCollateralizedbyParValue500ExchangeTradedCorporateDebtCoupon5.75MaturityDate202210Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LiabilitiesAndStockholdersEquity_4ae93f24-557b-4f5a-ac35-7eefc83b8046_totalLabel_en-US" xlink:label="lab_us-gaap_LiabilitiesAndStockholdersEquity" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Liabilities and Equity</link:label>
    <link:label id="lab_us-gaap_LiabilitiesAndStockholdersEquity_label_en-US" xlink:label="lab_us-gaap_LiabilitiesAndStockholdersEquity" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities and Equity</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LiabilitiesAndStockholdersEquity" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LiabilitiesAndStockholdersEquity"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LiabilitiesAndStockholdersEquity" xlink:to="lab_us-gaap_LiabilitiesAndStockholdersEquity" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansUnder250000Member_89f1ddb1-4183-44f1-8ba7-fc23494e82d7_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansUnder250000Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans under $250,000</link:label>
    <link:label id="lab_srt_MortgageLoansUnder250000Member_label_en-US" xlink:label="lab_srt_MortgageLoansUnder250000Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans under $250,000 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansUnder250000Member" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansUnder250000Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansUnder250000Member" xlink:to="lab_srt_MortgageLoansUnder250000Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncomeTaxDisclosureAbstract_11953edb-cef7-49d3-80fa-6f5dae9b6b9e_terseLabel_en-US" xlink:label="lab_us-gaap_IncomeTaxDisclosureAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Income Tax Disclosure [Abstract]</link:label>
    <link:label id="lab_us-gaap_IncomeTaxDisclosureAbstract_label_en-US" xlink:label="lab_us-gaap_IncomeTaxDisclosureAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Income Tax Disclosure [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncomeTaxDisclosureAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncomeTaxDisclosureAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncomeTaxDisclosureAbstract" xlink:to="lab_us-gaap_IncomeTaxDisclosureAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ConcentrationRiskPercentage1_a95cf78b-b7cc-4cb9-b3e0-15bded742183_terseLabel_en-US" xlink:label="lab_us-gaap_ConcentrationRiskPercentage1" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage of Total Outstanding Unpaid Principal Balance</link:label>
    <link:label id="lab_us-gaap_ConcentrationRiskPercentage1_label_en-US" xlink:label="lab_us-gaap_ConcentrationRiskPercentage1" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk, Percentage</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ConcentrationRiskPercentage1" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ConcentrationRiskPercentage1"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ConcentrationRiskPercentage1" xlink:to="lab_us-gaap_ConcentrationRiskPercentage1" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPercentMember_b07fc511-9c2c-46a5-b262-8501f78d576c_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Four Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Four Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Four Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2500000And2749999Member_f9ccb444-49a8-4e28-9d38-19e0762d6124_terseLabel_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2500000And2749999Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $2,500,000 and $2,749,999</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2500000And2749999Member_label_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2500000And2749999Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $2,500,000 and $2,749,999 [Member]</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2500000And2749999Member_documentation_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2500000And2749999Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $2,500,000 and $2,749,999</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2500000And2749999Member" xlink:href="efc-20201231.xsd#efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2500000And2749999Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2500000And2749999Member" xlink:to="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2500000And2749999Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InvestmentFairValueMeasurementInputDecimalType_4b9f0e90-9ad7-46f8-b66a-d71f98e4d315_terseLabel_en-US" xlink:label="lab_efc_InvestmentFairValueMeasurementInputDecimalType" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment, measurement input</link:label>
    <link:label id="lab_efc_InvestmentFairValueMeasurementInputDecimalType_label_en-US" xlink:label="lab_efc_InvestmentFairValueMeasurementInputDecimalType" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Fair Value Measurement Input, Decimal Type</link:label>
    <link:label id="lab_efc_InvestmentFairValueMeasurementInputDecimalType_documentation_en-US" xlink:label="lab_efc_InvestmentFairValueMeasurementInputDecimalType" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment fair value measurement input (investment can include but is not limited to securities, equity, loans, and real estate). Shown as a decimal.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InvestmentFairValueMeasurementInputDecimalType" xlink:href="efc-20201231.xsd#efc_InvestmentFairValueMeasurementInputDecimalType"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InvestmentFairValueMeasurementInputDecimalType" xlink:to="lab_efc_InvestmentFairValueMeasurementInputDecimalType" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtInstrumentTable_09bbb105-70ac-43d3-b0aa-b36213a7bdd2_terseLabel_en-US" xlink:label="lab_us-gaap_DebtInstrumentTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Long-term Debt Instruments [Table]</link:label>
    <link:label id="lab_us-gaap_DebtInstrumentTable_label_en-US" xlink:label="lab_us-gaap_DebtInstrumentTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Long-term Debt Instruments [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtInstrumentTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtInstrumentTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtInstrumentTable" xlink:to="lab_us-gaap_DebtInstrumentTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CorporateDebtSecuritiesMember_f104e489-f9ff-4aed-bbd8-82ba78c631ce_terseLabel_en-US" xlink:label="lab_us-gaap_CorporateDebtSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate debt securities</link:label>
    <link:label id="lab_us-gaap_CorporateDebtSecuritiesMember_label_en-US" xlink:label="lab_us-gaap_CorporateDebtSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate Debt Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CorporateDebtSecuritiesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CorporateDebtSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CorporateDebtSecuritiesMember" xlink:to="lab_us-gaap_CorporateDebtSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OtherOperatingIncome_6f48fb67-a88f-46cf-adf3-3606587ba70b_terseLabel_en-US" xlink:label="lab_us-gaap_OtherOperatingIncome" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other income</link:label>
    <link:label id="lab_us-gaap_OtherOperatingIncome_label_en-US" xlink:label="lab_us-gaap_OtherOperatingIncome" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Operating Income</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OtherOperatingIncome" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OtherOperatingIncome"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OtherOperatingIncome" xlink:to="lab_us-gaap_OtherOperatingIncome" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_DocumentFiscalYearFocus_c1481757-e785-498b-90b9-641d08f68757_terseLabel_en-US" xlink:label="lab_dei_DocumentFiscalYearFocus" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Document Fiscal Year Focus</link:label>
    <link:label id="lab_dei_DocumentFiscalYearFocus_label_en-US" xlink:label="lab_dei_DocumentFiscalYearFocus" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Document Fiscal Year Focus</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_DocumentFiscalYearFocus" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_DocumentFiscalYearFocus"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_DocumentFiscalYearFocus" xlink:to="lab_dei_DocumentFiscalYearFocus" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NotePurchaseAgreementJuly2019Member_8f2854cb-f824-4d36-aa5a-d419d06e6d01_terseLabel_en-US" xlink:label="lab_efc_NotePurchaseAgreementJuly2019Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Note Purchase Agreement July 2019 [Member]</link:label>
    <link:label id="lab_efc_NotePurchaseAgreementJuly2019Member_label_en-US" xlink:label="lab_efc_NotePurchaseAgreementJuly2019Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Note Purchase Agreement July 2019 [Member]</link:label>
    <link:label id="lab_efc_NotePurchaseAgreementJuly2019Member_documentation_en-US" xlink:label="lab_efc_NotePurchaseAgreementJuly2019Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Note purchase agreement (agreement to lend) executed in July 2019.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NotePurchaseAgreementJuly2019Member" xlink:href="efc-20201231.xsd#efc_NotePurchaseAgreementJuly2019Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NotePurchaseAgreementJuly2019Member" xlink:to="lab_efc_NotePurchaseAgreementJuly2019Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuredborrowingfacilitycollateralizedbyunsecuredloanportfolioMember_c1dfb7c0-c803-4dd8-92ef-eddf3bad8852_terseLabel_en-US" xlink:label="lab_efc_SecuredborrowingfacilitycollateralizedbyunsecuredloanportfolioMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Secured borrowing facility collateralized by unsecured loan portfolio</link:label>
    <link:label id="lab_efc_SecuredborrowingfacilitycollateralizedbyunsecuredloanportfolioMember_label_en-US" xlink:label="lab_efc_SecuredborrowingfacilitycollateralizedbyunsecuredloanportfolioMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Secured borrowing facility collateralized by unsecured loan portfolio [Member]</link:label>
    <link:label id="lab_efc_SecuredborrowingfacilitycollateralizedbyunsecuredloanportfolioMember_documentation_en-US" xlink:label="lab_efc_SecuredborrowingfacilitycollateralizedbyunsecuredloanportfolioMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Secured borrowing facility collateralized by unsecured loan portfolio [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuredborrowingfacilitycollateralizedbyunsecuredloanportfolioMember" xlink:href="efc-20201231.xsd#efc_SecuredborrowingfacilitycollateralizedbyunsecuredloanportfolioMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuredborrowingfacilitycollateralizedbyunsecuredloanportfolioMember" xlink:to="lab_efc_SecuredborrowingfacilitycollateralizedbyunsecuredloanportfolioMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CurrentStateAndLocalTaxExpenseBenefit_a88f4983-9355-4a6c-b0f5-6000fd58a02f_terseLabel_en-US" xlink:label="lab_us-gaap_CurrentStateAndLocalTaxExpenseBenefit" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Current State and Local Tax Expense (Benefit)</link:label>
    <link:label id="lab_us-gaap_CurrentStateAndLocalTaxExpenseBenefit_label_en-US" xlink:label="lab_us-gaap_CurrentStateAndLocalTaxExpenseBenefit" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Current State and Local Tax Expense (Benefit)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CurrentStateAndLocalTaxExpenseBenefit" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CurrentStateAndLocalTaxExpenseBenefit"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CurrentStateAndLocalTaxExpenseBenefit" xlink:to="lab_us-gaap_CurrentStateAndLocalTaxExpenseBenefit" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RetainednonriskretentionMember_353be26d-67c9-4c30-9cef-5d0878a0f2b2_terseLabel_en-US" xlink:label="lab_efc_RetainednonriskretentionMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Retained, non-risk retention [Member]</link:label>
    <link:label id="lab_efc_RetainednonriskretentionMember_label_en-US" xlink:label="lab_efc_RetainednonriskretentionMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Retained, non-risk retention [Member]</link:label>
    <link:label id="lab_efc_RetainednonriskretentionMember_documentation_en-US" xlink:label="lab_efc_RetainednonriskretentionMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Retained, non-risk retention [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RetainednonriskretentionMember" xlink:href="efc-20201231.xsd#efc_RetainednonriskretentionMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RetainednonriskretentionMember" xlink:to="lab_efc_RetainednonriskretentionMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProceedsFromSaleOfAvailableForSaleSecuritiesDebt_f69db16d-e6b1-4824-a1c8-434f05515bf1_verboseLabel_en-US" xlink:label="lab_us-gaap_ProceedsFromSaleOfAvailableForSaleSecuritiesDebt" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from disposition of securities</link:label>
    <link:label id="lab_us-gaap_ProceedsFromSaleOfAvailableForSaleSecuritiesDebt_label_en-US" xlink:label="lab_us-gaap_ProceedsFromSaleOfAvailableForSaleSecuritiesDebt" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from Sale of Debt Securities, Available-for-sale</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProceedsFromSaleOfAvailableForSaleSecuritiesDebt" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromSaleOfAvailableForSaleSecuritiesDebt"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProceedsFromSaleOfAvailableForSaleSecuritiesDebt" xlink:to="lab_us-gaap_ProceedsFromSaleOfAvailableForSaleSecuritiesDebt" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfStockByClassTable_9c191fd3-99d7-40ac-95cf-b110b57530ac_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfStockByClassTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Stock by Class [Table]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfStockByClassTable_label_en-US" xlink:label="lab_us-gaap_ScheduleOfStockByClassTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Stock by Class [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfStockByClassTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfStockByClassTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfStockByClassTable" xlink:to="lab_us-gaap_ScheduleOfStockByClassTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMarketCreditRiskMember_870e5582-4934-4e62-b740-8007cef681bd_terseLabel_en-US" xlink:label="lab_efc_EquityMarketCreditRiskMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Market/Credit Risk [Member]</link:label>
    <link:label id="lab_efc_EquityMarketCreditRiskMember_label_en-US" xlink:label="lab_efc_EquityMarketCreditRiskMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Market/Credit Risk [Member]</link:label>
    <link:label id="lab_efc_EquityMarketCreditRiskMember_documentation_en-US" xlink:label="lab_efc_EquityMarketCreditRiskMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Market/Credit Risk [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMarketCreditRiskMember" xlink:href="efc-20201231.xsd#efc_EquityMarketCreditRiskMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMarketCreditRiskMember" xlink:to="lab_efc_EquityMarketCreditRiskMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CSFirstBostonLimitedMember_4bafeefb-dad8-4954-9b40-94f46c66c73c_terseLabel_en-US" xlink:label="lab_efc_CSFirstBostonLimitedMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CS First Boston Limited [Member]</link:label>
    <link:label id="lab_efc_CSFirstBostonLimitedMember_label_en-US" xlink:label="lab_efc_CSFirstBostonLimitedMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CS First Boston Limited [Member]</link:label>
    <link:label id="lab_efc_CSFirstBostonLimitedMember_documentation_en-US" xlink:label="lab_efc_CSFirstBostonLimitedMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CS First Boston Limited [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CSFirstBostonLimitedMember" xlink:href="efc-20201231.xsd#efc_CSFirstBostonLimitedMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CSFirstBostonLimitedMember" xlink:to="lab_efc_CSFirstBostonLimitedMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RealestaterelatedMember_2d973a90-d865-4cfd-bc77-8320c58bc3ce_terseLabel_en-US" xlink:label="lab_efc_RealestaterelatedMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real estate-related [Member]</link:label>
    <link:label id="lab_efc_RealestaterelatedMember_label_en-US" xlink:label="lab_efc_RealestaterelatedMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real estate-related [Member]</link:label>
    <link:label id="lab_efc_RealestaterelatedMember_documentation_en-US" xlink:label="lab_efc_RealestaterelatedMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real estate-related [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RealestaterelatedMember" xlink:href="efc-20201231.xsd#efc_RealestaterelatedMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RealestaterelatedMember" xlink:to="lab_efc_RealestaterelatedMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFive_500d0ee4-2758-439f-a1e9-ccb5c8f21af4_terseLabel_en-US" xlink:label="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFive" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Maturities, Repayments of Principal after Year Five</link:label>
    <link:label id="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFive_label_en-US" xlink:label="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFive" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Maturities, Repayments of Principal after Year Five</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFive" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFive"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFive" xlink:to="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalAfterYearFive" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CreditDefaultSwapCorporateBondIndicesMember_d3365424-78f0-4448-8781-9e04e2b56137_terseLabel_en-US" xlink:label="lab_efc_CreditDefaultSwapCorporateBondIndicesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit default swaps on corporate bond indices</link:label>
    <link:label id="lab_efc_CreditDefaultSwapCorporateBondIndicesMember_label_en-US" xlink:label="lab_efc_CreditDefaultSwapCorporateBondIndicesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swap, Corporate Bond Indices [Member]</link:label>
    <link:label id="lab_efc_CreditDefaultSwapCorporateBondIndicesMember_documentation_en-US" xlink:label="lab_efc_CreditDefaultSwapCorporateBondIndicesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swap, Corporate Bond Indices [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CreditDefaultSwapCorporateBondIndicesMember" xlink:href="efc-20201231.xsd#efc_CreditDefaultSwapCorporateBondIndicesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CreditDefaultSwapCorporateBondIndicesMember" xlink:to="lab_efc_CreditDefaultSwapCorporateBondIndicesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_USGovernmentDebtSecuritiesMember_8c88fad4-d83c-44fb-a143-4909909d00c5_terseLabel_en-US" xlink:label="lab_us-gaap_USGovernmentDebtSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">US Government Debt Securities [Member]</link:label>
    <link:label id="lab_us-gaap_USGovernmentDebtSecuritiesMember_label_en-US" xlink:label="lab_us-gaap_USGovernmentDebtSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">US Government Debt Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_USGovernmentDebtSecuritiesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_USGovernmentDebtSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_USGovernmentDebtSecuritiesMember" xlink:to="lab_us-gaap_USGovernmentDebtSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RepaymentsOfSecuredDebt_0bc32b17-edec-4532-b5e6-8748e94d2e1e_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_RepaymentsOfSecuredDebt" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repayments of Secured Debt</link:label>
    <link:label id="lab_us-gaap_RepaymentsOfSecuredDebt_label_en-US" xlink:label="lab_us-gaap_RepaymentsOfSecuredDebt" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repayments of Secured Debt</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RepaymentsOfSecuredDebt" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RepaymentsOfSecuredDebt"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RepaymentsOfSecuredDebt" xlink:to="lab_us-gaap_RepaymentsOfSecuredDebt" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentCompanyFinancialHighlightsAbstract_label_en-US" xlink:label="lab_us-gaap_InvestmentCompanyFinancialHighlightsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Financial Highlights [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentCompanyFinancialHighlightsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentCompanyFinancialHighlightsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentCompanyFinancialHighlightsAbstract" xlink:to="lab_us-gaap_InvestmentCompanyFinancialHighlightsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SwapMember_d9f032c4-c41b-4385-829a-0585c5b6d768_terseLabel_en-US" xlink:label="lab_us-gaap_SwapMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Swap [Member]</link:label>
    <link:label id="lab_us-gaap_SwapMember_label_en-US" xlink:label="lab_us-gaap_SwapMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Swap [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SwapMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SwapMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SwapMember" xlink:to="lab_us-gaap_SwapMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AccountsPayableAndAccruedLiabilitiesCurrentAndNoncurrent_79575914-b2ac-4658-b4d7-8373f09c1e90_terseLabel_en-US" xlink:label="lab_us-gaap_AccountsPayableAndAccruedLiabilitiesCurrentAndNoncurrent" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accounts payable and accrued expenses</link:label>
    <link:label id="lab_us-gaap_AccountsPayableAndAccruedLiabilitiesCurrentAndNoncurrent_label_en-US" xlink:label="lab_us-gaap_AccountsPayableAndAccruedLiabilitiesCurrentAndNoncurrent" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accounts Payable and Accrued Liabilities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AccountsPayableAndAccruedLiabilitiesCurrentAndNoncurrent" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AccountsPayableAndAccruedLiabilitiesCurrentAndNoncurrent"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AccountsPayableAndAccruedLiabilitiesCurrentAndNoncurrent" xlink:to="lab_us-gaap_AccountsPayableAndAccruedLiabilitiesCurrentAndNoncurrent" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ForeignGovernmentDebtMember_eb9dc459-9881-42d0-bb22-fb7c1e3faf7f_terseLabel_en-US" xlink:label="lab_us-gaap_ForeignGovernmentDebtMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">European sovereign bonds</link:label>
    <link:label id="lab_us-gaap_ForeignGovernmentDebtMember_label_en-US" xlink:label="lab_us-gaap_ForeignGovernmentDebtMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Foreign Government Debt [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ForeignGovernmentDebtMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ForeignGovernmentDebtMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ForeignGovernmentDebtMember" xlink:to="lab_us-gaap_ForeignGovernmentDebtMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfCreditDerivativesTable_ec1faa23-81ef-4889-9b99-276016c61b05_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfCreditDerivativesTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Credit Derivatives [Table]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfCreditDerivativesTable_label_en-US" xlink:label="lab_us-gaap_ScheduleOfCreditDerivativesTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Credit Derivatives [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfCreditDerivativesTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfCreditDerivativesTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfCreditDerivativesTable" xlink:to="lab_us-gaap_ScheduleOfCreditDerivativesTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year4Point00PercentMember_71b42f6f-9909-4964-9faf-7cfae7a35f04_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year4Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 4 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year4Point00PercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year4Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 4 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year4Point00PercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year4Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 4.00%</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociation30Year4Point00PercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociation30Year4Point00PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociation30Year4Point00PercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociation30Year4Point00PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OtherinvestmentrelatedexpensesAbstract_f50ee1a4-592a-4b6b-a9a1-08159a83d74c_terseLabel_en-US" xlink:label="lab_efc_OtherinvestmentrelatedexpensesAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other investment related expenses</link:label>
    <link:label id="lab_efc_OtherinvestmentrelatedexpensesAbstract_label_en-US" xlink:label="lab_efc_OtherinvestmentrelatedexpensesAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other investment related expenses [Abstract]</link:label>
    <link:label id="lab_efc_OtherinvestmentrelatedexpensesAbstract_documentation_en-US" xlink:label="lab_efc_OtherinvestmentrelatedexpensesAbstract" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other investment related expenses [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OtherinvestmentrelatedexpensesAbstract" xlink:href="efc-20201231.xsd#efc_OtherinvestmentrelatedexpensesAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OtherinvestmentrelatedexpensesAbstract" xlink:to="lab_efc_OtherinvestmentrelatedexpensesAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CommonStockSoldShortbyindustryMember_80f2f57e-07cd-4c3f-8297-f525f3fb2a63_terseLabel_en-US" xlink:label="lab_efc_CommonStockSoldShortbyindustryMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common stock</link:label>
    <link:label id="lab_efc_CommonStockSoldShortbyindustryMember_label_en-US" xlink:label="lab_efc_CommonStockSoldShortbyindustryMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common Stock Sold Short, by industry [Member]</link:label>
    <link:label id="lab_efc_CommonStockSoldShortbyindustryMember_documentation_en-US" xlink:label="lab_efc_CommonStockSoldShortbyindustryMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">This element provides for investments in common stock sold short required to be summarized by the following industries: (1) public utilities; (2) banks, trust and insurance companies; (3) industrial, miscellaneous and all other.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CommonStockSoldShortbyindustryMember" xlink:href="efc-20201231.xsd#efc_CommonStockSoldShortbyindustryMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CommonStockSoldShortbyindustryMember" xlink:to="lab_efc_CommonStockSoldShortbyindustryMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AwardTypeAxis_6a0b0504-a560-4f96-afb6-d2d055fd260a_terseLabel_en-US" xlink:label="lab_us-gaap_AwardTypeAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Award Type [Axis]</link:label>
    <link:label id="lab_us-gaap_AwardTypeAxis_label_en-US" xlink:label="lab_us-gaap_AwardTypeAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Award Type [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AwardTypeAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AwardTypeAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AwardTypeAxis" xlink:to="lab_us-gaap_AwardTypeAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Netrealizedgainlossoncurrencyhedges_69f7dac4-4e61-45dc-8e8e-77e55d8fcb7e_terseLabel_en-US" xlink:label="lab_efc_Netrealizedgainlossoncurrencyhedges" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial derivatives-currency hedges</link:label>
    <link:label id="lab_efc_Netrealizedgainlossoncurrencyhedges_label_en-US" xlink:label="lab_efc_Netrealizedgainlossoncurrencyhedges" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net realized gain (loss) on currency hedges</link:label>
    <link:label id="lab_efc_Netrealizedgainlossoncurrencyhedges_documentation_en-US" xlink:label="lab_efc_Netrealizedgainlossoncurrencyhedges" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net realized gain (loss) on currency hedges</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Netrealizedgainlossoncurrencyhedges" xlink:href="efc-20201231.xsd#efc_Netrealizedgainlossoncurrencyhedges"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Netrealizedgainlossoncurrencyhedges" xlink:to="lab_efc_Netrealizedgainlossoncurrencyhedges" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DebtInstrumentMeasurementInputPercent_e0eddf58-3661-4574-bb4d-1679833842ea_terseLabel_en-US" xlink:label="lab_efc_DebtInstrumentMeasurementInputPercent" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt, measurement input</link:label>
    <link:label id="lab_efc_DebtInstrumentMeasurementInputPercent_label_en-US" xlink:label="lab_efc_DebtInstrumentMeasurementInputPercent" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Instrument, Measurement Input, Percent</link:label>
    <link:label id="lab_efc_DebtInstrumentMeasurementInputPercent_documentation_en-US" xlink:label="lab_efc_DebtInstrumentMeasurementInputPercent" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Value of input used to measure debt instrument, including, but not limited to, convertible and non-convertible debt.Shown as a percent.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DebtInstrumentMeasurementInputPercent" xlink:href="efc-20201231.xsd#efc_DebtInstrumentMeasurementInputPercent"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DebtInstrumentMeasurementInputPercent" xlink:to="lab_efc_DebtInstrumentMeasurementInputPercent" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonperformingCommercialMortgageLoansandCommercialRealEstateOwnedMember_068c0b29-1d2a-4ba6-a2b2-2f0e5a933310_terseLabel_en-US" xlink:label="lab_efc_NonperformingCommercialMortgageLoansandCommercialRealEstateOwnedMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-performing commercial mortgage loans and commercial real estate owned</link:label>
    <link:label id="lab_efc_NonperformingCommercialMortgageLoansandCommercialRealEstateOwnedMember_label_en-US" xlink:label="lab_efc_NonperformingCommercialMortgageLoansandCommercialRealEstateOwnedMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-performing Commercial Mortgage Loans and Commercial Real Estate Owned [Member]</link:label>
    <link:label id="lab_efc_NonperformingCommercialMortgageLoansandCommercialRealEstateOwnedMember_documentation_en-US" xlink:label="lab_efc_NonperformingCommercialMortgageLoansandCommercialRealEstateOwnedMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-performing Commercial Mortgage Loans and Commercial Real Estate Owned [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonperformingCommercialMortgageLoansandCommercialRealEstateOwnedMember" xlink:href="efc-20201231.xsd#efc_NonperformingCommercialMortgageLoansandCommercialRealEstateOwnedMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonperformingCommercialMortgageLoansandCommercialRealEstateOwnedMember" xlink:to="lab_efc_NonperformingCommercialMortgageLoansandCommercialRealEstateOwnedMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeLiabilityFairValueGrossAsset_215adc82-74b5-421e-9823-693afb173ca4_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeLiabilityFairValueGrossAsset" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Liability, Fair Value, Gross Asset</link:label>
    <link:label id="lab_us-gaap_DerivativeLiabilityFairValueGrossAsset_label_en-US" xlink:label="lab_us-gaap_DerivativeLiabilityFairValueGrossAsset" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Liability, Fair Value, Gross Asset</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeLiabilityFairValueGrossAsset" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeLiabilityFairValueGrossAsset"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeLiabilityFairValueGrossAsset" xlink:to="lab_us-gaap_DerivativeLiabilityFairValueGrossAsset" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPointFivePercentMember_e6c57153-248e-454b-a452-d905d43b98ed_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Four Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPointFivePercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFourPointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CommercialPropertyMember_be74dc01-dd26-4408-9499-1f940039da37_terseLabel_en-US" xlink:label="lab_efc_CommercialPropertyMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commercial Property [Member]</link:label>
    <link:label id="lab_efc_CommercialPropertyMember_label_en-US" xlink:label="lab_efc_CommercialPropertyMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commercial Property [Member]</link:label>
    <link:label id="lab_efc_CommercialPropertyMember_documentation_en-US" xlink:label="lab_efc_CommercialPropertyMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commercial Property [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CommercialPropertyMember" xlink:href="efc-20201231.xsd#efc_CommercialPropertyMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CommercialPropertyMember" xlink:to="lab_efc_CommercialPropertyMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleofUnvestedLTIPUnitsTable_b372eca5-3c70-489e-8124-65b12223fa63_terseLabel_en-US" xlink:label="lab_efc_ScheduleofUnvestedLTIPUnitsTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Unvested LTIP Units [Table]</link:label>
    <link:label id="lab_efc_ScheduleofUnvestedLTIPUnitsTable_label_en-US" xlink:label="lab_efc_ScheduleofUnvestedLTIPUnitsTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Unvested LTIP Units [Table]</link:label>
    <link:label id="lab_efc_ScheduleofUnvestedLTIPUnitsTable_documentation_en-US" xlink:label="lab_efc_ScheduleofUnvestedLTIPUnitsTable" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Unvested LTIP Units [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleofUnvestedLTIPUnitsTable" xlink:href="efc-20201231.xsd#efc_ScheduleofUnvestedLTIPUnitsTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleofUnvestedLTIPUnitsTable" xlink:to="lab_efc_ScheduleofUnvestedLTIPUnitsTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentIncomeTextBlock_2925907f-0ce6-44f8-acea-93bbed51f304_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentIncomeTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Income</link:label>
    <link:label id="lab_us-gaap_InvestmentIncomeTextBlock_label_en-US" xlink:label="lab_us-gaap_InvestmentIncomeTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Income [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentIncomeTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentIncomeTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentIncomeTextBlock" xlink:to="lab_us-gaap_InvestmentIncomeTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityAccretedDiscountsAmortizedPremiums_497e3891-1f4b-446d-a119-11c31c5b13ab_terseLabel_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityAccretedDiscountsAmortizedPremiums" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities, Accreted Discounts/Amortized Premiums</link:label>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityAccretedDiscountsAmortizedPremiums_label_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityAccretedDiscountsAmortizedPremiums" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement With Unobservable Inputs Reconciliations, Recurring Basis, Liability, Accreted Discounts/Amortized Premiums</link:label>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityAccretedDiscountsAmortizedPremiums_documentation_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityAccretedDiscountsAmortizedPremiums" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement With Unobservable Inputs Reconciliations, Recurring Basis, Liability, Accreted Discounts/Amortized Premiums</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityAccretedDiscountsAmortizedPremiums" xlink:href="efc-20201231.xsd#efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityAccretedDiscountsAmortizedPremiums"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityAccretedDiscountsAmortizedPremiums" xlink:to="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityAccretedDiscountsAmortizedPremiums" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NetIncomeLossAttributableToNoncontrollingInterest_07929698-bb6f-48d4-8fa7-9ee76812f108_terseLabel_en-US" xlink:label="lab_us-gaap_NetIncomeLossAttributableToNoncontrollingInterest" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net income (loss) attributable to non-controlling interests</link:label>
    <link:label id="lab_us-gaap_NetIncomeLossAttributableToNoncontrollingInterest_label_en-US" xlink:label="lab_us-gaap_NetIncomeLossAttributableToNoncontrollingInterest" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Income (Loss) Attributable to Noncontrolling Interest</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NetIncomeLossAttributableToNoncontrollingInterest" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NetIncomeLossAttributableToNoncontrollingInterest"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NetIncomeLossAttributableToNoncontrollingInterest" xlink:to="lab_us-gaap_NetIncomeLossAttributableToNoncontrollingInterest" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancingReceivableAllowanceForCreditLosses_7224e968-feaf-4d5d-84fa-6f21517071a8_terseLabel_en-US" xlink:label="lab_us-gaap_FinancingReceivableAllowanceForCreditLosses" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable, Allowance for Credit Loss</link:label>
    <link:label id="lab_us-gaap_FinancingReceivableAllowanceForCreditLosses_label_en-US" xlink:label="lab_us-gaap_FinancingReceivableAllowanceForCreditLosses" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable, Allowance for Credit Loss</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancingReceivableAllowanceForCreditLosses" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancingReceivableAllowanceForCreditLosses"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancingReceivableAllowanceForCreditLosses" xlink:to="lab_us-gaap_FinancingReceivableAllowanceForCreditLosses" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Repaymentsofrepurchaseagreementnoncash_31c34ea8-5d3e-4ffb-8606-6bba7cfe983c_terseLabel_en-US" xlink:label="lab_efc_Repaymentsofrepurchaseagreementnoncash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repayments of repurchase agreement (non-cash)</link:label>
    <link:label id="lab_efc_Repaymentsofrepurchaseagreementnoncash_label_en-US" xlink:label="lab_efc_Repaymentsofrepurchaseagreementnoncash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repayments of repurchase agreement (non-cash)</link:label>
    <link:label id="lab_efc_Repaymentsofrepurchaseagreementnoncash_documentation_en-US" xlink:label="lab_efc_Repaymentsofrepurchaseagreementnoncash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-cash repayments of borrowings from securities sold under agreements to repurchase.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Repaymentsofrepurchaseagreementnoncash" xlink:href="efc-20201231.xsd#efc_Repaymentsofrepurchaseagreementnoncash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Repaymentsofrepurchaseagreementnoncash" xlink:to="lab_efc_Repaymentsofrepurchaseagreementnoncash" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Maturity61to90daysMember_eda64731-872e-4dad-a44b-78866f7efd1d_terseLabel_en-US" xlink:label="lab_efc_Maturity61to90daysMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">61-90 Days</link:label>
    <link:label id="lab_efc_Maturity61to90daysMember_label_en-US" xlink:label="lab_efc_Maturity61to90daysMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity 61 to 90 days [Member]</link:label>
    <link:label id="lab_efc_Maturity61to90daysMember_documentation_en-US" xlink:label="lab_efc_Maturity61to90daysMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity 61 to 90 days [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Maturity61to90daysMember" xlink:href="efc-20201231.xsd#efc_Maturity61to90daysMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Maturity61to90daysMember" xlink:to="lab_efc_Maturity61to90daysMember" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_OH_fb94da95-1455-4cb8-a15c-e87b35970352_terseLabel_en-US" xlink:label="lab_stpr_OH" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">OHIO</link:label>
    <link:label id="lab_stpr_OH_label_en-US" xlink:label="lab_stpr_OH" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">OHIO</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_OH" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_OH"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_OH" xlink:to="lab_stpr_OH" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NumberOfRealEstateProperties_e66000f7-e347-496e-81d3-9cf15688077f_terseLabel_en-US" xlink:label="lab_us-gaap_NumberOfRealEstateProperties" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of Real Estate Properties</link:label>
    <link:label id="lab_us-gaap_NumberOfRealEstateProperties_7958c9bb-594a-4c19-a30c-6db17d38ba0f_periodStartLabel_en-US" xlink:label="lab_us-gaap_NumberOfRealEstateProperties" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Beginning Balance</link:label>
    <link:label id="lab_us-gaap_NumberOfRealEstateProperties_08ee4bf2-f625-469d-85c6-80b157b2bfde_periodEndLabel_en-US" xlink:label="lab_us-gaap_NumberOfRealEstateProperties" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ending Balance</link:label>
    <link:label id="lab_us-gaap_NumberOfRealEstateProperties_label_en-US" xlink:label="lab_us-gaap_NumberOfRealEstateProperties" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of Real Estate Properties</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NumberOfRealEstateProperties" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NumberOfRealEstateProperties"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NumberOfRealEstateProperties" xlink:to="lab_us-gaap_NumberOfRealEstateProperties" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_StatementGeographicalAxis_b4cefa4b-b3ae-45e7-a4df-410927d88a0c_terseLabel_en-US" xlink:label="lab_srt_StatementGeographicalAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Geographical [Axis]</link:label>
    <link:label id="lab_srt_StatementGeographicalAxis_label_en-US" xlink:label="lab_srt_StatementGeographicalAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Geographical [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_StatementGeographicalAxis" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_StatementGeographicalAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_StatementGeographicalAxis" xlink:to="lab_srt_StatementGeographicalAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DeferredTaxAssetsInvestments_cdbf9d4d-7f1b-4fd7-8674-318717a9df83_terseLabel_en-US" xlink:label="lab_us-gaap_DeferredTaxAssetsInvestments" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Tax Assets, Investments</link:label>
    <link:label id="lab_us-gaap_DeferredTaxAssetsInvestments_label_en-US" xlink:label="lab_us-gaap_DeferredTaxAssetsInvestments" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Tax Assets, Investments</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DeferredTaxAssetsInvestments" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DeferredTaxAssetsInvestments"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DeferredTaxAssetsInvestments" xlink:to="lab_us-gaap_DeferredTaxAssetsInvestments" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Nonperformingloansmaturitydatenotapplicable_2b88c8cb-991b-4d5e-a5f4-19c8467df5bd_terseLabel_en-US" xlink:label="lab_efc_Nonperformingloansmaturitydatenotapplicable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-performing loans, maturity date not applicable</link:label>
    <link:label id="lab_efc_Nonperformingloansmaturitydatenotapplicable_label_en-US" xlink:label="lab_efc_Nonperformingloansmaturitydatenotapplicable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-performing loans, maturity date not applicable</link:label>
    <link:label id="lab_efc_Nonperformingloansmaturitydatenotapplicable_documentation_en-US" xlink:label="lab_efc_Nonperformingloansmaturitydatenotapplicable" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair value of non-performing loans where a maturity date is not applicable and this is not included in the disclosed date range.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Nonperformingloansmaturitydatenotapplicable" xlink:href="efc-20201231.xsd#efc_Nonperformingloansmaturitydatenotapplicable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Nonperformingloansmaturitydatenotapplicable" xlink:to="lab_efc_Nonperformingloansmaturitydatenotapplicable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentSoldNotYetPurchasedPercentOfNetAssets_3c489557-04e3-44ee-a900-b91fc29159c4_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_InvestmentSoldNotYetPurchasedPercentOfNetAssets" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Sold, Not yet Purchased, Percent of Net Assets</link:label>
    <link:label id="lab_us-gaap_InvestmentSoldNotYetPurchasedPercentOfNetAssets_label_en-US" xlink:label="lab_us-gaap_InvestmentSoldNotYetPurchasedPercentOfNetAssets" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Sold, Not yet Purchased, Percent of Net Assets</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentSoldNotYetPurchasedPercentOfNetAssets" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentSoldNotYetPurchasedPercentOfNetAssets"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentSoldNotYetPurchasedPercentOfNetAssets" xlink:to="lab_us-gaap_InvestmentSoldNotYetPurchasedPercentOfNetAssets" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RetainedEarningsMember_e1e7e361-0a95-46a1-9c99-6191a0955fa0_terseLabel_en-US" xlink:label="lab_us-gaap_RetainedEarningsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Retained Earnings/(Accumulated Deficit)</link:label>
    <link:label id="lab_us-gaap_RetainedEarningsMember_label_en-US" xlink:label="lab_us-gaap_RetainedEarningsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Retained Earnings [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RetainedEarningsMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RetainedEarningsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RetainedEarningsMember" xlink:to="lab_us-gaap_RetainedEarningsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansOnRealEstateOriginalLoanAmountDomain_0c92f47c-ab7e-4eac-b5e9-fbf32a14c9a5_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateOriginalLoanAmountDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Original Loan Amount [Domain]</link:label>
    <link:label id="lab_srt_MortgageLoansOnRealEstateOriginalLoanAmountDomain_label_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateOriginalLoanAmountDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Original Loan Amount [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansOnRealEstateOriginalLoanAmountDomain" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansOnRealEstateOriginalLoanAmountDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansOnRealEstateOriginalLoanAmountDomain" xlink:to="lab_srt_MortgageLoansOnRealEstateOriginalLoanAmountDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OtherEquityMethodInvestmentsMember_62dee824-b3a1-417f-a38e-b7bb0ef133a9_terseLabel_en-US" xlink:label="lab_efc_OtherEquityMethodInvestmentsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Equity Method Investments [Member]</link:label>
    <link:label id="lab_efc_OtherEquityMethodInvestmentsMember_label_en-US" xlink:label="lab_efc_OtherEquityMethodInvestmentsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Equity Method Investments [Member]</link:label>
    <link:label id="lab_efc_OtherEquityMethodInvestmentsMember_documentation_en-US" xlink:label="lab_efc_OtherEquityMethodInvestmentsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Equity Method Investments [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OtherEquityMethodInvestmentsMember" xlink:href="efc-20201231.xsd#efc_OtherEquityMethodInvestmentsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OtherEquityMethodInvestmentsMember" xlink:to="lab_efc_OtherEquityMethodInvestmentsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ForeignCurrencyTransactionGainLossRealized_e2e6e8d9-d42a-45e1-aa3a-2ce6a4855c45_terseLabel_en-US" xlink:label="lab_us-gaap_ForeignCurrencyTransactionGainLossRealized" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Foreign currency transactions</link:label>
    <link:label id="lab_us-gaap_ForeignCurrencyTransactionGainLossRealized_6751c6ea-a4ce-460a-8deb-074b59b9f763_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_ForeignCurrencyTransactionGainLossRealized" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized (gains) losses other, net&#8212;foreign currency transaction</link:label>
    <link:label id="lab_us-gaap_ForeignCurrencyTransactionGainLossRealized_label_en-US" xlink:label="lab_us-gaap_ForeignCurrencyTransactionGainLossRealized" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Foreign Currency Transaction Gain (Loss), Realized</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ForeignCurrencyTransactionGainLossRealized" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ForeignCurrencyTransactionGainLossRealized"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ForeignCurrencyTransactionGainLossRealized" xlink:to="lab_us-gaap_ForeignCurrencyTransactionGainLossRealized" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SubsequentEventTypeDomain_b432354a-0399-4ec6-91fa-24e675eab989_terseLabel_en-US" xlink:label="lab_us-gaap_SubsequentEventTypeDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsequent Event Type [Domain]</link:label>
    <link:label id="lab_us-gaap_SubsequentEventTypeDomain_label_en-US" xlink:label="lab_us-gaap_SubsequentEventTypeDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsequent Event Type [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SubsequentEventTypeDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SubsequentEventTypeDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SubsequentEventTypeDomain" xlink:to="lab_us-gaap_SubsequentEventTypeDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMarketRiskMember_01838598-e86b-4195-9369-7e635ff4ee99_terseLabel_en-US" xlink:label="lab_efc_EquityMarketRiskMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Market Risk [Member]</link:label>
    <link:label id="lab_efc_EquityMarketRiskMember_label_en-US" xlink:label="lab_efc_EquityMarketRiskMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Market Risk [Member]</link:label>
    <link:label id="lab_efc_EquityMarketRiskMember_documentation_en-US" xlink:label="lab_efc_EquityMarketRiskMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Market Risk [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMarketRiskMember" xlink:href="efc-20201231.xsd#efc_EquityMarketRiskMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMarketRiskMember" xlink:to="lab_efc_EquityMarketRiskMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ManagementFeeExpenseRebates_a665d70a-a592-46fb-ab0e-e98674160578_terseLabel_en-US" xlink:label="lab_efc_ManagementFeeExpenseRebates" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Management Fee Expense, Rebates</link:label>
    <link:label id="lab_efc_ManagementFeeExpenseRebates_label_en-US" xlink:label="lab_efc_ManagementFeeExpenseRebates" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Management Fee Expense, Rebates</link:label>
    <link:label id="lab_efc_ManagementFeeExpenseRebates_documentation_en-US" xlink:label="lab_efc_ManagementFeeExpenseRebates" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Management Fee Expense, Rebates</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ManagementFeeExpenseRebates" xlink:href="efc-20201231.xsd#efc_ManagementFeeExpenseRebates"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ManagementFeeExpenseRebates" xlink:to="lab_efc_ManagementFeeExpenseRebates" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_TransferorDomain_873cf686-f582-4c88-a319-6cabce7ff1d4_terseLabel_en-US" xlink:label="lab_efc_TransferorDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transferor [Domain]</link:label>
    <link:label id="lab_efc_TransferorDomain_label_en-US" xlink:label="lab_efc_TransferorDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transferor [Domain]</link:label>
    <link:label id="lab_efc_TransferorDomain_documentation_en-US" xlink:label="lab_efc_TransferorDomain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Domain] for Transferor [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_TransferorDomain" xlink:href="efc-20201231.xsd#efc_TransferorDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_TransferorDomain" xlink:to="lab_efc_TransferorDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_NY_cccaa6ea-257f-43d5-ad99-4023e78db48b_verboseLabel_en-US" xlink:label="lab_stpr_NY" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maryland [Member]</link:label>
    <link:label id="lab_stpr_NY_5ca942b4-ac38-4a14-bd82-237334b5a6ba_terseLabel_en-US" xlink:label="lab_stpr_NY" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">NEW YORK</link:label>
    <link:label id="lab_stpr_NY_label_en-US" xlink:label="lab_stpr_NY" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">NEW YORK</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_NY" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_NY"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_NY" xlink:to="lab_stpr_NY" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MinimumPercentageOfIncentiveFeeToBePaidInShare_f5e84a2c-2b96-4d54-ba78-6754a3b0fae8_terseLabel_en-US" xlink:label="lab_efc_MinimumPercentageOfIncentiveFeeToBePaidInShare" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Minimum percentage of incentive fee to be paid in share</link:label>
    <link:label id="lab_efc_MinimumPercentageOfIncentiveFeeToBePaidInShare_label_en-US" xlink:label="lab_efc_MinimumPercentageOfIncentiveFeeToBePaidInShare" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Minimum percentage of incentive fee to be paid in share</link:label>
    <link:label id="lab_efc_MinimumPercentageOfIncentiveFeeToBePaidInShare_documentation_en-US" xlink:label="lab_efc_MinimumPercentageOfIncentiveFeeToBePaidInShare" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The minimum percentage of the incentive fee to be paid to the Manager in the form of common shares.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MinimumPercentageOfIncentiveFeeToBePaidInShare" xlink:href="efc-20201231.xsd#efc_MinimumPercentageOfIncentiveFeeToBePaidInShare"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MinimumPercentageOfIncentiveFeeToBePaidInShare" xlink:to="lab_efc_MinimumPercentageOfIncentiveFeeToBePaidInShare" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NotesissuedbyCLOsecuritization_8698b0d0-03d5-45d5-997e-98590a4d0e36_terseLabel_en-US" xlink:label="lab_efc_NotesissuedbyCLOsecuritization" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Face Amount of Notes Issued</link:label>
    <link:label id="lab_efc_NotesissuedbyCLOsecuritization_label_en-US" xlink:label="lab_efc_NotesissuedbyCLOsecuritization" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Notes issued by CLO securitization</link:label>
    <link:label id="lab_efc_NotesissuedbyCLOsecuritization_documentation_en-US" xlink:label="lab_efc_NotesissuedbyCLOsecuritization" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Original face amount of notes issued pursuant to a CLO securitization.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NotesissuedbyCLOsecuritization" xlink:href="efc-20201231.xsd#efc_NotesissuedbyCLOsecuritization"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NotesissuedbyCLOsecuritization" xlink:to="lab_efc_NotesissuedbyCLOsecuritization" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_TerminationFeePeriod_63a5c30b-2529-49a7-a4be-b8be7db78e62_terseLabel_en-US" xlink:label="lab_efc_TerminationFeePeriod" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Termination Fee, Period</link:label>
    <link:label id="lab_efc_TerminationFeePeriod_label_en-US" xlink:label="lab_efc_TerminationFeePeriod" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Termination Fee, Period</link:label>
    <link:label id="lab_efc_TerminationFeePeriod_documentation_en-US" xlink:label="lab_efc_TerminationFeePeriod" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Termination Fee, Period</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_TerminationFeePeriod" xlink:href="efc-20201231.xsd#efc_TerminationFeePeriod"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_TerminationFeePeriod" xlink:to="lab_efc_TerminationFeePeriod" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_NV_703f9436-e2df-4ef6-8e44-c19fcf506f8b_terseLabel_en-US" xlink:label="lab_stpr_NV" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">NEVADA</link:label>
    <link:label id="lab_stpr_NV_label_en-US" xlink:label="lab_stpr_NV" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">NEVADA</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_NV" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_NV"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_NV" xlink:to="lab_stpr_NV" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_TotalInvestmentsAndFinancialDerivatives_b7a9c394-92d4-44be-81a1-c73ace45ad17_terseLabel_en-US" xlink:label="lab_efc_TotalInvestmentsAndFinancialDerivatives" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total investments and financial derivatives</link:label>
    <link:label id="lab_efc_TotalInvestmentsAndFinancialDerivatives_label_en-US" xlink:label="lab_efc_TotalInvestmentsAndFinancialDerivatives" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total investments and financial derivatives</link:label>
    <link:label id="lab_efc_TotalInvestmentsAndFinancialDerivatives_documentation_en-US" xlink:label="lab_efc_TotalInvestmentsAndFinancialDerivatives" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Investments And Financial Derivatives.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_TotalInvestmentsAndFinancialDerivatives" xlink:href="efc-20201231.xsd#efc_TotalInvestmentsAndFinancialDerivatives"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_TotalInvestmentsAndFinancialDerivatives" xlink:to="lab_efc_TotalInvestmentsAndFinancialDerivatives" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_DocumentPeriodEndDate_de183fde-cec4-4932-8237-95b0f2b200d8_terseLabel_en-US" xlink:label="lab_dei_DocumentPeriodEndDate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Document Period End Date</link:label>
    <link:label id="lab_dei_DocumentPeriodEndDate_label_en-US" xlink:label="lab_dei_DocumentPeriodEndDate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Document Period End Date</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_DocumentPeriodEndDate" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_DocumentPeriodEndDate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_DocumentPeriodEndDate" xlink:to="lab_dei_DocumentPeriodEndDate" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_WeightedAverageNumberOfSharesOutstandingDilutedDisclosureItemsAbstract_5154fdd5-a982-4bdf-adc3-4500dc397e6e_terseLabel_en-US" xlink:label="lab_us-gaap_WeightedAverageNumberOfSharesOutstandingDilutedDisclosureItemsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Weighted average shares outstanding (basic and diluted):</link:label>
    <link:label id="lab_us-gaap_WeightedAverageNumberOfSharesOutstandingDilutedDisclosureItemsAbstract_label_en-US" xlink:label="lab_us-gaap_WeightedAverageNumberOfSharesOutstandingDilutedDisclosureItemsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Weighted Average Number of Shares Outstanding Reconciliation [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_WeightedAverageNumberOfSharesOutstandingDilutedDisclosureItemsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_WeightedAverageNumberOfSharesOutstandingDilutedDisclosureItemsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_WeightedAverageNumberOfSharesOutstandingDilutedDisclosureItemsAbstract" xlink:to="lab_us-gaap_WeightedAverageNumberOfSharesOutstandingDilutedDisclosureItemsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StatementTable_faffaf2a-08ba-4825-9685-4643b98045df_terseLabel_en-US" xlink:label="lab_us-gaap_StatementTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Statement [Table]</link:label>
    <link:label id="lab_us-gaap_StatementTable_label_en-US" xlink:label="lab_us-gaap_StatementTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Statement [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StatementTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StatementTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StatementTable" xlink:to="lab_us-gaap_StatementTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreementsUnsettled_d8ce71f0-967f-497b-9d9c-f03e715fb143_terseLabel_en-US" xlink:label="lab_efc_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreementsUnsettled" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Pledged Financial Instruments, Not Separately Reported, Securities For Repurchase Agreements Unsettled</link:label>
    <link:label id="lab_efc_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreementsUnsettled_label_en-US" xlink:label="lab_efc_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreementsUnsettled" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Pledged Financial Instruments, Not Separately Reported, Securities For Repurchase Agreements Unsettled</link:label>
    <link:label id="lab_efc_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreementsUnsettled_documentation_en-US" xlink:label="lab_efc_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreementsUnsettled" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The carrying amount as of the date of the latest financial statement presented of securities which are owned but transferred to serve as collateral for repurchase agreements which have been sold prior to period end but remain unsettled at the balance sheet date.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreementsUnsettled" xlink:href="efc-20201231.xsd#efc_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreementsUnsettled"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreementsUnsettled" xlink:to="lab_efc_PledgedFinancialInstrumentsNotSeparatelyReportedSecuritiesForRepurchaseAgreementsUnsettled" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InterestRateCapMember_6411d7c1-9527-402f-8499-a538b76575df_terseLabel_en-US" xlink:label="lab_us-gaap_InterestRateCapMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Rate Cap [Member]</link:label>
    <link:label id="lab_us-gaap_InterestRateCapMember_label_en-US" xlink:label="lab_us-gaap_InterestRateCapMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Rate Cap [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InterestRateCapMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InterestRateCapMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InterestRateCapMember" xlink:to="lab_us-gaap_InterestRateCapMember" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityRegistrantName_b36ea0a8-3f85-4578-8a53-3dc2dae15897_terseLabel_en-US" xlink:label="lab_dei_EntityRegistrantName" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Registrant Name</link:label>
    <link:label id="lab_dei_EntityRegistrantName_label_en-US" xlink:label="lab_dei_EntityRegistrantName" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Registrant Name</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityRegistrantName" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityRegistrantName"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityRegistrantName" xlink:to="lab_dei_EntityRegistrantName" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_EquityMethodInvesteeNameDomain_902b91f5-b63c-4fe4-90dc-0a7dc8c3596c_terseLabel_en-US" xlink:label="lab_srt_EquityMethodInvesteeNameDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment, Name [Domain]</link:label>
    <link:label id="lab_srt_EquityMethodInvesteeNameDomain_label_en-US" xlink:label="lab_srt_EquityMethodInvesteeNameDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment, Name [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_EquityMethodInvesteeNameDomain" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_EquityMethodInvesteeNameDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_EquityMethodInvesteeNameDomain" xlink:to="lab_srt_EquityMethodInvesteeNameDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CommercialMortgageBackedSecuritiesMember_161bac3b-bf2e-4b12-a24b-e45a86931be0_terseLabel_en-US" xlink:label="lab_us-gaap_CommercialMortgageBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CMBS</link:label>
    <link:label id="lab_us-gaap_CommercialMortgageBackedSecuritiesMember_label_en-US" xlink:label="lab_us-gaap_CommercialMortgageBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commercial Mortgage Backed Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CommercialMortgageBackedSecuritiesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CommercialMortgageBackedSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CommercialMortgageBackedSecuritiesMember" xlink:to="lab_us-gaap_CommercialMortgageBackedSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LongInvestmentBbbRatingMember_181b8070-fead-49d3-a8b9-a83df83e6f2e_terseLabel_en-US" xlink:label="lab_efc_LongInvestmentBbbRatingMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Bbb Rating [Member]</link:label>
    <link:label id="lab_efc_LongInvestmentBbbRatingMember_label_en-US" xlink:label="lab_efc_LongInvestmentBbbRatingMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Bbb Rating [Member]</link:label>
    <link:label id="lab_efc_LongInvestmentBbbRatingMember_documentation_en-US" xlink:label="lab_efc_LongInvestmentBbbRatingMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">BBB rating as derived from Standard and Poor's, Moody's, and/or Fitch.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LongInvestmentBbbRatingMember" xlink:href="efc-20201231.xsd#efc_LongInvestmentBbbRatingMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LongInvestmentBbbRatingMember" xlink:to="lab_efc_LongInvestmentBbbRatingMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentOwnedPercentOfNetAssets_0286eea9-772f-42b0-bd99-a64191edfb19_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentOwnedPercentOfNetAssets" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Owned, Percent of Net Assets</link:label>
    <link:label id="lab_us-gaap_InvestmentOwnedPercentOfNetAssets_label_en-US" xlink:label="lab_us-gaap_InvestmentOwnedPercentOfNetAssets" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Owned, Percent of Net Assets</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentOwnedPercentOfNetAssets" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentOwnedPercentOfNetAssets"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentOwnedPercentOfNetAssets" xlink:to="lab_us-gaap_InvestmentOwnedPercentOfNetAssets" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_NC_384bca1b-35d0-491e-bc13-2e18adadc4be_terseLabel_en-US" xlink:label="lab_stpr_NC" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">NORTH CAROLINA</link:label>
    <link:label id="lab_stpr_NC_578f98fe-b125-4d2e-8545-16d2d1eff23d_verboseLabel_en-US" xlink:label="lab_stpr_NC" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">PENNSYLVANIA</link:label>
    <link:label id="lab_stpr_NC_label_en-US" xlink:label="lab_stpr_NC" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">NORTH CAROLINA</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_NC" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_NC"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_NC" xlink:to="lab_stpr_NC" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue3400U.S.TreasuryNoteCoupon3.13MaturityDate202811Member_5aea1729-a708-4fdf-bd5e-ba36bb699331_terseLabel_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue3400U.S.TreasuryNoteCoupon3.13MaturityDate202811Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $3,400 U.S. Treasury Note, Coupon 3.13%, Maturity Date 2028 11 [Member]</link:label>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue3400U.S.TreasuryNoteCoupon3.13MaturityDate202811Member_label_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue3400U.S.TreasuryNoteCoupon3.13MaturityDate202811Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $3,400 U.S. Treasury Note, Coupon 3.13%, Maturity Date 2028 11 [Member]</link:label>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue3400U.S.TreasuryNoteCoupon3.13MaturityDate202811Member_documentation_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue3400U.S.TreasuryNoteCoupon3.13MaturityDate202811Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $3,400 U.S. Treasury Note, Coupon 3.13%, Maturity Date 2028 11 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_JPMorganSecuritiesLLCCollateralizedbyParValue3400U.S.TreasuryNoteCoupon3.13MaturityDate202811Member" xlink:href="efc-20201231.xsd#efc_JPMorganSecuritiesLLCCollateralizedbyParValue3400U.S.TreasuryNoteCoupon3.13MaturityDate202811Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_JPMorganSecuritiesLLCCollateralizedbyParValue3400U.S.TreasuryNoteCoupon3.13MaturityDate202811Member" xlink:to="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue3400U.S.TreasuryNoteCoupon3.13MaturityDate202811Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationMember_61822bab-9bb0-479a-8504-84475aef10b1_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationMember" xlink:to="lab_efc_FederalNationalMortgageAssociationMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Adjustmentstononcontrollinginterests_071e5452-ade4-447b-a7eb-48744416ca2f_terseLabel_en-US" xlink:label="lab_efc_Adjustmentstononcontrollinginterests" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Adjustment to non-controlling interests</link:label>
    <link:label id="lab_efc_Adjustmentstononcontrollinginterests_label_en-US" xlink:label="lab_efc_Adjustmentstononcontrollinginterests" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Adjustments to non-controlling interests</link:label>
    <link:label id="lab_efc_Adjustmentstononcontrollinginterests_documentation_en-US" xlink:label="lab_efc_Adjustmentstononcontrollinginterests" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Adjustments to non-controlling interests</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Adjustmentstononcontrollinginterests" xlink:href="efc-20201231.xsd#efc_Adjustmentstononcontrollinginterests"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Adjustmentstononcontrollinginterests" xlink:to="lab_efc_Adjustmentstononcontrollinginterests" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfFairValueAssetsAndLiabilitiesMeasuredOnRecurringBasisTableTextBlock_4d695d20-1f40-4f86-84eb-dc0a94d70014_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfFairValueAssetsAndLiabilitiesMeasuredOnRecurringBasisTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Fair Value, Assets and Liabilities Measured on Recurring Basis [Table Text Block]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfFairValueAssetsAndLiabilitiesMeasuredOnRecurringBasisTableTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfFairValueAssetsAndLiabilitiesMeasuredOnRecurringBasisTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Fair Value, Assets and Liabilities Measured on Recurring Basis [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfFairValueAssetsAndLiabilitiesMeasuredOnRecurringBasisTableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfFairValueAssetsAndLiabilitiesMeasuredOnRecurringBasisTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfFairValueAssetsAndLiabilitiesMeasuredOnRecurringBasisTableTextBlock" xlink:to="lab_us-gaap_ScheduleOfFairValueAssetsAndLiabilitiesMeasuredOnRecurringBasisTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InterestAndDividendsPayableCurrentAndNoncurrent_a568a4ee-c747-458e-88ad-8e6d2a4cc1f6_terseLabel_en-US" xlink:label="lab_us-gaap_InterestAndDividendsPayableCurrentAndNoncurrent" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest and Dividends Payable</link:label>
    <link:label id="lab_us-gaap_InterestAndDividendsPayableCurrentAndNoncurrent_label_en-US" xlink:label="lab_us-gaap_InterestAndDividendsPayableCurrentAndNoncurrent" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest and Dividends Payable</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InterestAndDividendsPayableCurrentAndNoncurrent" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InterestAndDividendsPayableCurrentAndNoncurrent"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InterestAndDividendsPayableCurrentAndNoncurrent" xlink:to="lab_us-gaap_InterestAndDividendsPayableCurrentAndNoncurrent" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPercentMember_f742b50b-75b0-4ef9-aee3-ba1243c101c4_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Fifteen Year) Four Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Fifteen Year) Four Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Fifteen Year) Four Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsFifteenYearFourPercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_AccretiveDilutiveEffectofShareIssuancesNetofOfferingCostsShareRepurchasesandAdjustmentstoNoncontrollinginterests_b495f485-73cd-4c3e-85aa-378e3ece053e_terseLabel_en-US" xlink:label="lab_efc_AccretiveDilutiveEffectofShareIssuancesNetofOfferingCostsShareRepurchasesandAdjustmentstoNoncontrollinginterests" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accretive (Dilutive) Effect of Share Issuances (Net of Offering Costs), Share Repurchases, and Adjustments to Non-controlling interests</link:label>
    <link:label id="lab_efc_AccretiveDilutiveEffectofShareIssuancesNetofOfferingCostsShareRepurchasesandAdjustmentstoNoncontrollinginterests_label_en-US" xlink:label="lab_efc_AccretiveDilutiveEffectofShareIssuancesNetofOfferingCostsShareRepurchasesandAdjustmentstoNoncontrollinginterests" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accretive (Dilutive) Effect of Share Issuances (Net of Offering Costs), Share Repurchases, and Adjustments to Non-controlling interests</link:label>
    <link:label id="lab_efc_AccretiveDilutiveEffectofShareIssuancesNetofOfferingCostsShareRepurchasesandAdjustmentstoNoncontrollinginterests_documentation_en-US" xlink:label="lab_efc_AccretiveDilutiveEffectofShareIssuancesNetofOfferingCostsShareRepurchasesandAdjustmentstoNoncontrollinginterests" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Per share impact of share issuances/repurchases, or adjustments to non-controlling interests.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_AccretiveDilutiveEffectofShareIssuancesNetofOfferingCostsShareRepurchasesandAdjustmentstoNoncontrollinginterests" xlink:href="efc-20201231.xsd#efc_AccretiveDilutiveEffectofShareIssuancesNetofOfferingCostsShareRepurchasesandAdjustmentstoNoncontrollinginterests"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_AccretiveDilutiveEffectofShareIssuancesNetofOfferingCostsShareRepurchasesandAdjustmentstoNoncontrollinginterests" xlink:to="lab_efc_AccretiveDilutiveEffectofShareIssuancesNetofOfferingCostsShareRepurchasesandAdjustmentstoNoncontrollinginterests" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MaximumMember_e563998d-bd0f-47fd-9702-e3f672dff51a_terseLabel_en-US" xlink:label="lab_srt_MaximumMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maximum</link:label>
    <link:label id="lab_srt_MaximumMember_label_en-US" xlink:label="lab_srt_MaximumMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maximum [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MaximumMember" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MaximumMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MaximumMember" xlink:to="lab_srt_MaximumMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentCompanyContractualFeeWaived_61437651-1c6a-4399-b75c-1241ea767941_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentCompanyContractualFeeWaived" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Management fee rebate</link:label>
    <link:label id="lab_us-gaap_InvestmentCompanyContractualFeeWaived_label_en-US" xlink:label="lab_us-gaap_InvestmentCompanyContractualFeeWaived" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Contractual Fee Waived</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentCompanyContractualFeeWaived" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentCompanyContractualFeeWaived"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentCompanyContractualFeeWaived" xlink:to="lab_us-gaap_InvestmentCompanyContractualFeeWaived" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_NJ_adab7df1-1648-4ca8-9e03-da48106262b7_terseLabel_en-US" xlink:label="lab_stpr_NJ" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">NEW JERSEY</link:label>
    <link:label id="lab_stpr_NJ_9ed50e08-d271-4e73-bbc1-0fde28e05a6e_verboseLabel_en-US" xlink:label="lab_stpr_NJ" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">MASSACHUSETTS</link:label>
    <link:label id="lab_stpr_NJ_label_en-US" xlink:label="lab_stpr_NJ" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">NEW JERSEY</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_NJ" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_NJ"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_NJ" xlink:to="lab_stpr_NJ" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LoansNotesTradeAndOtherReceivablesDisclosureTextBlock_a9d47d11-2156-4d0a-a77d-7b5bbf4cfed4_terseLabel_en-US" xlink:label="lab_us-gaap_LoansNotesTradeAndOtherReceivablesDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment in Loans</link:label>
    <link:label id="lab_us-gaap_LoansNotesTradeAndOtherReceivablesDisclosureTextBlock_label_en-US" xlink:label="lab_us-gaap_LoansNotesTradeAndOtherReceivablesDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans, Notes, Trade and Other Receivables Disclosure [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LoansNotesTradeAndOtherReceivablesDisclosureTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LoansNotesTradeAndOtherReceivablesDisclosureTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LoansNotesTradeAndOtherReceivablesDisclosureTextBlock" xlink:to="lab_us-gaap_LoansNotesTradeAndOtherReceivablesDisclosureTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_MO_0d29c15b-e95f-4b38-a620-140ffd551455_terseLabel_en-US" xlink:label="lab_stpr_MO" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">NORTH CAROLINA</link:label>
    <link:label id="lab_stpr_MO_label_en-US" xlink:label="lab_stpr_MO" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">MISSOURI</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_MO" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_MO"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_MO" xlink:to="lab_stpr_MO" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_AvailableforsalesecuritiesRealizesLossesWriteoffs_0d315b5e-81b3-4f59-a94c-f979c6bd208c_terseLabel_en-US" xlink:label="lab_efc_AvailableforsalesecuritiesRealizesLossesWriteoffs" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Available-for-sale securities Realizes Losses - Write offs</link:label>
    <link:label id="lab_efc_AvailableforsalesecuritiesRealizesLossesWriteoffs_label_en-US" xlink:label="lab_efc_AvailableforsalesecuritiesRealizesLossesWriteoffs" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Available-for-sale securities Realizes Losses - Write offs</link:label>
    <link:label id="lab_efc_AvailableforsalesecuritiesRealizesLossesWriteoffs_documentation_en-US" xlink:label="lab_efc_AvailableforsalesecuritiesRealizesLossesWriteoffs" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized losses resulting from a complete or partial write off once the Company has determined all or a portion of an impaired securities cost basis is uncollectible.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_AvailableforsalesecuritiesRealizesLossesWriteoffs" xlink:href="efc-20201231.xsd#efc_AvailableforsalesecuritiesRealizesLossesWriteoffs"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_AvailableforsalesecuritiesRealizesLossesWriteoffs" xlink:to="lab_efc_AvailableforsalesecuritiesRealizesLossesWriteoffs" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes_a6bed324-13d5-4872-b8ad-7e3c75cb5e62_terseLabel_en-US" xlink:label="lab_us-gaap_EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Effective Income Tax Rate Reconciliation, State and Local Income Taxes, Percent</link:label>
    <link:label id="lab_us-gaap_EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes_label_en-US" xlink:label="lab_us-gaap_EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Effective Income Tax Rate Reconciliation, State and Local Income Taxes, Percent</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes" xlink:to="lab_us-gaap_EffectiveIncomeTaxRateReconciliationStateAndLocalIncomeTaxes" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CreditMember_13eb7e4b-31ce-47bc-9a74-aa2889249d8f_terseLabel_en-US" xlink:label="lab_us-gaap_CreditMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit [Member]</link:label>
    <link:label id="lab_us-gaap_CreditMember_label_en-US" xlink:label="lab_us-gaap_CreditMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CreditMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CreditMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CreditMember" xlink:to="lab_us-gaap_CreditMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSalePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate_4d965d7c-3605-4199-aff2-9c95d0b3dde1_terseLabel_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSalePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Purchased with Credit Deterioration, Allowance for Credit Loss at Acquisition Date</link:label>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSalePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate_label_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSalePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Purchased with Credit Deterioration, Allowance for Credit Loss at Acquisition Date</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtSecuritiesAvailableForSalePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSalePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtSecuritiesAvailableForSalePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate" xlink:to="lab_us-gaap_DebtSecuritiesAvailableForSalePurchasedWithCreditDeteriorationAllowanceForCreditLossAtAcquisitionDate" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfFinancingReceivablesPastDueTable_f8ffeb24-e7b5-4e4f-ab97-5a4a84f95a05_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfFinancingReceivablesPastDueTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable, Past Due [Table]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfFinancingReceivablesPastDueTable_label_en-US" xlink:label="lab_us-gaap_ScheduleOfFinancingReceivablesPastDueTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable, Past Due [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfFinancingReceivablesPastDueTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfFinancingReceivablesPastDueTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfFinancingReceivablesPastDueTable" xlink:to="lab_us-gaap_ScheduleOfFinancingReceivablesPastDueTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ReceivablesAbstract_24c16c2f-5d4e-4644-804e-2e9341222033_terseLabel_en-US" xlink:label="lab_us-gaap_ReceivablesAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Receivables [Abstract]</link:label>
    <link:label id="lab_us-gaap_ReceivablesAbstract_label_en-US" xlink:label="lab_us-gaap_ReceivablesAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Receivables [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ReceivablesAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ReceivablesAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ReceivablesAbstract" xlink:to="lab_us-gaap_ReceivablesAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsTwentyYearFourPercentMember_c1fe48c6-a9f3-40ca-ac0e-767e03dfa8c4_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsTwentyYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Twenty Year) Four Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsTwentyYearFourPercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsTwentyYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Twenty Year) Four Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsTwentyYearFourPercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsTwentyYearFourPercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Twenty Year) Four Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsTwentyYearFourPercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsTwentyYearFourPercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsTwentyYearFourPercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsTwentyYearFourPercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PercentageofTotalCollateralOnReverseRepurchaseAgreements_dad49371-1257-40c8-933e-0a58671b2c81_terseLabel_en-US" xlink:label="lab_efc_PercentageofTotalCollateralOnReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage of Total Collateral On Reverse Repurchase Agreements</link:label>
    <link:label id="lab_efc_PercentageofTotalCollateralOnReverseRepurchaseAgreements_label_en-US" xlink:label="lab_efc_PercentageofTotalCollateralOnReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage of Total Collateral On Reverse Repurchase Agreements</link:label>
    <link:label id="lab_efc_PercentageofTotalCollateralOnReverseRepurchaseAgreements_documentation_en-US" xlink:label="lab_efc_PercentageofTotalCollateralOnReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage of total collateral on reverse repurchase agreements held by a specific counterparty that is above a specified concentration risk threshold.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PercentageofTotalCollateralOnReverseRepurchaseAgreements" xlink:href="efc-20201231.xsd#efc_PercentageofTotalCollateralOnReverseRepurchaseAgreements"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PercentageofTotalCollateralOnReverseRepurchaseAgreements" xlink:to="lab_efc_PercentageofTotalCollateralOnReverseRepurchaseAgreements" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999999Member_50e537b3-07ba-4e2d-8065-a41eb4718367_terseLabel_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999999Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $5,000,000 and $9,999,999</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999999Member_label_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999999Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $5,000,000 and $9,999,999 [Member]</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999999Member_documentation_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999999Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $5,000,000 and $9,999,999</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999999Member" xlink:href="efc-20201231.xsd#efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999999Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999999Member" xlink:to="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween5000000And9999999Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CommercialmortgageloansandREOMember_8e933d73-c8a6-4044-adb8-3d75ecf0eb6d_terseLabel_en-US" xlink:label="lab_efc_CommercialmortgageloansandREOMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commercial mortgage loans and REO</link:label>
    <link:label id="lab_efc_CommercialmortgageloansandREOMember_label_en-US" xlink:label="lab_efc_CommercialmortgageloansandREOMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commercial mortgage loans and REO [Member]</link:label>
    <link:label id="lab_efc_CommercialmortgageloansandREOMember_documentation_en-US" xlink:label="lab_efc_CommercialmortgageloansandREOMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commercial mortgage loans and REO</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CommercialmortgageloansandREOMember" xlink:href="efc-20201231.xsd#efc_CommercialmortgageloansandREOMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CommercialmortgageloansandREOMember" xlink:to="lab_efc_CommercialmortgageloansandREOMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OperatingPartnershipLTIPUnitsDistributed_df63cfe6-9ebb-4c84-99bb-87456ba5d4d4_terseLabel_en-US" xlink:label="lab_efc_OperatingPartnershipLTIPUnitsDistributed" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership LTIP Units Distributed (in units)</link:label>
    <link:label id="lab_efc_OperatingPartnershipLTIPUnitsDistributed_label_en-US" xlink:label="lab_efc_OperatingPartnershipLTIPUnitsDistributed" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership LTIP Units Distributed</link:label>
    <link:label id="lab_efc_OperatingPartnershipLTIPUnitsDistributed_documentation_en-US" xlink:label="lab_efc_OperatingPartnershipLTIPUnitsDistributed" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership LTIP Units Distributed</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OperatingPartnershipLTIPUnitsDistributed" xlink:href="efc-20201231.xsd#efc_OperatingPartnershipLTIPUnitsDistributed"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OperatingPartnershipLTIPUnitsDistributed" xlink:to="lab_efc_OperatingPartnershipLTIPUnitsDistributed" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_U.S.TreasuryFuturesMember_830cbf7e-0420-4396-ad6e-92484214b9c9_terseLabel_en-US" xlink:label="lab_efc_U.S.TreasuryFuturesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury futures</link:label>
    <link:label id="lab_efc_U.S.TreasuryFuturesMember_label_en-US" xlink:label="lab_efc_U.S.TreasuryFuturesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Futures [Member]</link:label>
    <link:label id="lab_efc_U.S.TreasuryFuturesMember_documentation_en-US" xlink:label="lab_efc_U.S.TreasuryFuturesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Futures [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_U.S.TreasuryFuturesMember" xlink:href="efc-20201231.xsd#efc_U.S.TreasuryFuturesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_U.S.TreasuryFuturesMember" xlink:to="lab_efc_U.S.TreasuryFuturesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_OwnershipAxis_104e5d63-d802-47eb-a54b-de1069a757be_terseLabel_en-US" xlink:label="lab_srt_OwnershipAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ownership [Axis]</link:label>
    <link:label id="lab_srt_OwnershipAxis_label_en-US" xlink:label="lab_srt_OwnershipAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ownership [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_OwnershipAxis" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_OwnershipAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_OwnershipAxis" xlink:to="lab_srt_OwnershipAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityPublicFloat_970bec92-05d1-454f-ae19-87d411570781_terseLabel_en-US" xlink:label="lab_dei_EntityPublicFloat" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Public Float</link:label>
    <link:label id="lab_dei_EntityPublicFloat_label_en-US" xlink:label="lab_dei_EntityPublicFloat" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Public Float</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityPublicFloat" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityPublicFloat"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityPublicFloat" xlink:to="lab_dei_EntityPublicFloat" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InvestmentpurchasedfromrelatedpartyMember_44e7c0f8-76bd-4b9d-947d-88498437e3ea_terseLabel_en-US" xlink:label="lab_efc_InvestmentpurchasedfromrelatedpartyMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment purchased from related party</link:label>
    <link:label id="lab_efc_InvestmentpurchasedfromrelatedpartyMember_label_en-US" xlink:label="lab_efc_InvestmentpurchasedfromrelatedpartyMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment purchased from related party [Member]</link:label>
    <link:label id="lab_efc_InvestmentpurchasedfromrelatedpartyMember_documentation_en-US" xlink:label="lab_efc_InvestmentpurchasedfromrelatedpartyMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment purchased from related party [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InvestmentpurchasedfromrelatedpartyMember" xlink:href="efc-20201231.xsd#efc_InvestmentpurchasedfromrelatedpartyMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InvestmentpurchasedfromrelatedpartyMember" xlink:to="lab_efc_InvestmentpurchasedfromrelatedpartyMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeLiabilityFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection_065a6139-1a0a-4a5f-b36b-d402d3eaf212_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_DerivativeLiabilityFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Amount</link:label>
    <link:label id="lab_us-gaap_DerivativeLiabilityFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection_label_en-US" xlink:label="lab_us-gaap_DerivativeLiabilityFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Liability, Fair Value, Offset Against Collateral, Net of Not Subject to Master Netting Arrangement, Policy Election</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeLiabilityFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeLiabilityFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeLiabilityFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:to="lab_us-gaap_DerivativeLiabilityFairValueOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LongInvestmentAaaRatingMember_335df4c2-2f34-470a-97ae-d5780a3acbf6_terseLabel_en-US" xlink:label="lab_efc_LongInvestmentAaaRatingMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Aaa Rating [Member]</link:label>
    <link:label id="lab_efc_LongInvestmentAaaRatingMember_label_en-US" xlink:label="lab_efc_LongInvestmentAaaRatingMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Aaa Rating [Member]</link:label>
    <link:label id="lab_efc_LongInvestmentAaaRatingMember_documentation_en-US" xlink:label="lab_efc_LongInvestmentAaaRatingMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Aaa Rating [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LongInvestmentAaaRatingMember" xlink:href="efc-20201231.xsd#efc_LongInvestmentAaaRatingMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LongInvestmentAaaRatingMember" xlink:to="lab_efc_LongInvestmentAaaRatingMember" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_MD_7f2971cc-a7c5-4ed6-97c5-d9dc1a2097d7_terseLabel_en-US" xlink:label="lab_stpr_MD" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">MARYLAND</link:label>
    <link:label id="lab_stpr_MD_label_en-US" xlink:label="lab_stpr_MD" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">MARYLAND</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_MD" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_MD"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_MD" xlink:to="lab_stpr_MD" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTextBlock_0f5f1562-e742-4984-9f4e-39d1289dfd14_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Summary of Investments in Loans</link:label>
    <link:label id="lab_us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Accounts, Notes, Loans and Financing Receivable [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTextBlock" xlink:to="lab_us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancialInstrumentPerformanceStatusDomain_69cc1ca9-9584-43d8-ba65-e5ee12216a4f_terseLabel_en-US" xlink:label="lab_us-gaap_FinancialInstrumentPerformanceStatusDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instrument Performance Status [Domain]</link:label>
    <link:label id="lab_us-gaap_FinancialInstrumentPerformanceStatusDomain_label_en-US" xlink:label="lab_us-gaap_FinancialInstrumentPerformanceStatusDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instrument Performance Status [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancialInstrumentPerformanceStatusDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancialInstrumentPerformanceStatusDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancialInstrumentPerformanceStatusDomain" xlink:to="lab_us-gaap_FinancialInstrumentPerformanceStatusDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CashEquivalentsMember_cffb1329-d212-4f08-9f63-7fd9920bce86_terseLabel_en-US" xlink:label="lab_us-gaap_CashEquivalentsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash equivalents</link:label>
    <link:label id="lab_us-gaap_CashEquivalentsMember_label_en-US" xlink:label="lab_us-gaap_CashEquivalentsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash Equivalents [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CashEquivalentsMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CashEquivalentsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CashEquivalentsMember" xlink:to="lab_us-gaap_CashEquivalentsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMethodInvestmentsSummarizedFinancialInformationOtherassets_040811d2-92b7-4dc6-ba6d-32b07206c0fa_terseLabel_en-US" xlink:label="lab_efc_EquityMethodInvestmentsSummarizedFinancialInformationOtherassets" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investments, Summarized Financial Information, Other assets</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentsSummarizedFinancialInformationOtherassets_label_en-US" xlink:label="lab_efc_EquityMethodInvestmentsSummarizedFinancialInformationOtherassets" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investments, Summarized Financial Information, Other assets</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentsSummarizedFinancialInformationOtherassets_documentation_en-US" xlink:label="lab_efc_EquityMethodInvestmentsSummarizedFinancialInformationOtherassets" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The amount of other assets reported by an equity method investment of the entity.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMethodInvestmentsSummarizedFinancialInformationOtherassets" xlink:href="efc-20201231.xsd#efc_EquityMethodInvestmentsSummarizedFinancialInformationOtherassets"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMethodInvestmentsSummarizedFinancialInformationOtherassets" xlink:to="lab_efc_EquityMethodInvestmentsSummarizedFinancialInformationOtherassets" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_MA_65532684-cfce-440c-a88a-1c14602f92e3_terseLabel_en-US" xlink:label="lab_stpr_MA" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">MASSACHUSETTS</link:label>
    <link:label id="lab_stpr_MA_label_en-US" xlink:label="lab_stpr_MA" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">MASSACHUSETTS</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_MA" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_MA"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_MA" xlink:to="lab_stpr_MA" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_QuarterlyFinancialInformationTextBlock_ffd54e46-cf07-4c06-9b20-8a154f2a456a_terseLabel_en-US" xlink:label="lab_us-gaap_QuarterlyFinancialInformationTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Quarterly Financial Information</link:label>
    <link:label id="lab_us-gaap_QuarterlyFinancialInformationTextBlock_label_en-US" xlink:label="lab_us-gaap_QuarterlyFinancialInformationTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Quarterly Financial Information [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_QuarterlyFinancialInformationTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_QuarterlyFinancialInformationTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_QuarterlyFinancialInformationTextBlock" xlink:to="lab_us-gaap_QuarterlyFinancialInformationTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_AggregateToBeAnnouncedTradeActivityBuysSellsNonCash_4ad6dc75-3092-4c34-bb50-4534dcce2315_terseLabel_en-US" xlink:label="lab_efc_AggregateToBeAnnouncedTradeActivityBuysSellsNonCash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Aggregate To Be Announced Trade Activity Buys Sells Non Cash</link:label>
    <link:label id="lab_efc_AggregateToBeAnnouncedTradeActivityBuysSellsNonCash_label_en-US" xlink:label="lab_efc_AggregateToBeAnnouncedTradeActivityBuysSellsNonCash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Aggregate To Be Announced Trade Activity Buys Sells Non Cash</link:label>
    <link:label id="lab_efc_AggregateToBeAnnouncedTradeActivityBuysSellsNonCash_documentation_en-US" xlink:label="lab_efc_AggregateToBeAnnouncedTradeActivityBuysSellsNonCash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Represents the absolute value of To-Be-Announced mortgage pass-through certificates or "TBAs" transactions inclusive of TBA roll transactions and non-roll purchases and sales, all having closed or settled during the period.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_AggregateToBeAnnouncedTradeActivityBuysSellsNonCash" xlink:href="efc-20201231.xsd#efc_AggregateToBeAnnouncedTradeActivityBuysSellsNonCash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_AggregateToBeAnnouncedTradeActivityBuysSellsNonCash" xlink:to="lab_efc_AggregateToBeAnnouncedTradeActivityBuysSellsNonCash" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Paymentsonfinancialderivatives_2266a03c-118b-4ae9-b5fc-187ef943be38_negatedTerseLabel_en-US" xlink:label="lab_efc_Paymentsonfinancialderivatives" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments on financial derivatives</link:label>
    <link:label id="lab_efc_Paymentsonfinancialderivatives_label_en-US" xlink:label="lab_efc_Paymentsonfinancialderivatives" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments on financial derivatives</link:label>
    <link:label id="lab_efc_Paymentsonfinancialderivatives_documentation_en-US" xlink:label="lab_efc_Paymentsonfinancialderivatives" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments on financial derivatives</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Paymentsonfinancialderivatives" xlink:href="efc-20201231.xsd#efc_Paymentsonfinancialderivatives"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Paymentsonfinancialderivatives" xlink:to="lab_efc_Paymentsonfinancialderivatives" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentSoldNotYetPurchasedSaleProceeds_b3661b48-1be9-475d-bcec-8736f626c59e_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentSoldNotYetPurchasedSaleProceeds" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments sold short, at fair value, Proceeds</link:label>
    <link:label id="lab_us-gaap_InvestmentSoldNotYetPurchasedSaleProceeds_21c5792b-b82a-4b8c-b695-73fa3532bc99_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_InvestmentSoldNotYetPurchasedSaleProceeds" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Sold, Not yet Purchased, Sale Proceeds</link:label>
    <link:label id="lab_us-gaap_InvestmentSoldNotYetPurchasedSaleProceeds_label_en-US" xlink:label="lab_us-gaap_InvestmentSoldNotYetPurchasedSaleProceeds" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Sold, Not yet Purchased, Sale Proceeds</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentSoldNotYetPurchasedSaleProceeds" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentSoldNotYetPurchasedSaleProceeds"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentSoldNotYetPurchasedSaleProceeds" xlink:to="lab_us-gaap_InvestmentSoldNotYetPurchasedSaleProceeds" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2020Member_5486b4b3-b9f9-48c7-a913-2852b3638e75_terseLabel_en-US" xlink:label="lab_efc_A2020Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2020 [Member]</link:label>
    <link:label id="lab_efc_A2020Member_label_en-US" xlink:label="lab_efc_A2020Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2020 [Member]</link:label>
    <link:label id="lab_efc_A2020Member_documentation_en-US" xlink:label="lab_efc_A2020Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2020 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2020Member" xlink:href="efc-20201231.xsd#efc_A2020Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2020Member" xlink:to="lab_efc_A2020Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfAvailableForSaleSecuritiesTable_0f0bd9fc-c80e-447c-9933-8fee44e2384f_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfAvailableForSaleSecuritiesTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Available-for-sale Securities [Table]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfAvailableForSaleSecuritiesTable_label_en-US" xlink:label="lab_us-gaap_ScheduleOfAvailableForSaleSecuritiesTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Available-for-sale Securities [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfAvailableForSaleSecuritiesTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfAvailableForSaleSecuritiesTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfAvailableForSaleSecuritiesTable" xlink:to="lab_us-gaap_ScheduleOfAvailableForSaleSecuritiesTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ProceedsFromTheDispositionOfLoansNonCash_dd48f07a-d464-498c-9b15-909bbe46db23_terseLabel_en-US" xlink:label="lab_efc_ProceedsFromTheDispositionOfLoansNonCash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from the disposition of loans (non-cash)</link:label>
    <link:label id="lab_efc_ProceedsFromTheDispositionOfLoansNonCash_label_en-US" xlink:label="lab_efc_ProceedsFromTheDispositionOfLoansNonCash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from the disposition of loans (non-cash)</link:label>
    <link:label id="lab_efc_ProceedsFromTheDispositionOfLoansNonCash_documentation_en-US" xlink:label="lab_efc_ProceedsFromTheDispositionOfLoansNonCash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from the disposition of loans (non-cash)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ProceedsFromTheDispositionOfLoansNonCash" xlink:href="efc-20201231.xsd#efc_ProceedsFromTheDispositionOfLoansNonCash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ProceedsFromTheDispositionOfLoansNonCash" xlink:to="lab_efc_ProceedsFromTheDispositionOfLoansNonCash" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RelatedpartyjointventureinterestMember_9a2f4ff5-ba8d-45b1-a1e0-cf29932ffcb2_terseLabel_en-US" xlink:label="lab_efc_RelatedpartyjointventureinterestMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related party joint venture interest</link:label>
    <link:label id="lab_efc_RelatedpartyjointventureinterestMember_label_en-US" xlink:label="lab_efc_RelatedpartyjointventureinterestMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related party joint venture interest [Member]</link:label>
    <link:label id="lab_efc_RelatedpartyjointventureinterestMember_documentation_en-US" xlink:label="lab_efc_RelatedpartyjointventureinterestMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related party joint venture interest [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RelatedpartyjointventureinterestMember" xlink:href="efc-20201231.xsd#efc_RelatedpartyjointventureinterestMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RelatedpartyjointventureinterestMember" xlink:to="lab_efc_RelatedpartyjointventureinterestMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsOtherThreePointFivePercentMember_ed0dd547-8673-4e00-bb53-054a52417052_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsOtherThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Other) Three Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsOtherThreePointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsOtherThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Other) Three Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsOtherThreePointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsOtherThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Other) Three Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationPoolsOtherThreePointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationPoolsOtherThreePointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationPoolsOtherThreePointFivePercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationPoolsOtherThreePointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Maturity121to150daysMember_5e110343-2955-4cbc-980e-2f4d6c123b5d_terseLabel_en-US" xlink:label="lab_efc_Maturity121to150daysMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">121-150 Days</link:label>
    <link:label id="lab_efc_Maturity121to150daysMember_label_en-US" xlink:label="lab_efc_Maturity121to150daysMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity 121 to 150 days [Member]</link:label>
    <link:label id="lab_efc_Maturity121to150daysMember_documentation_en-US" xlink:label="lab_efc_Maturity121to150daysMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity 121 to 150 days [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Maturity121to150daysMember" xlink:href="efc-20201231.xsd#efc_Maturity121to150daysMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Maturity121to150daysMember" xlink:to="lab_efc_Maturity121to150daysMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtInstrumentFaceAmount_2bd967aa-8b4d-4810-b9de-2cdb233508fc_terseLabel_en-US" xlink:label="lab_us-gaap_DebtInstrumentFaceAmount" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Instrument, Face Amount</link:label>
    <link:label id="lab_us-gaap_DebtInstrumentFaceAmount_label_en-US" xlink:label="lab_us-gaap_DebtInstrumentFaceAmount" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Instrument, Face Amount</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtInstrumentFaceAmount" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtInstrumentFaceAmount"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtInstrumentFaceAmount" xlink:to="lab_us-gaap_DebtInstrumentFaceAmount" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporation30Year4Point00PercentMember_1ebf0be2-5222-45a9-bc0e-9133445934c8_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporation30Year4Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation (30 Year) 4 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporation30Year4Point00PercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporation30Year4Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation (30 Year) 4 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporation30Year4Point00PercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporation30Year4Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation (30 Year) 4.00% [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporation30Year4Point00PercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporation30Year4Point00PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporation30Year4Point00PercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporation30Year4Point00PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NetRealizedGainsLossesonPeriodicSettlementsofInterestRateSwaps_0aafee55-59b4-4183-a850-7e3ca99d62cf_terseLabel_en-US" xlink:label="lab_efc_NetRealizedGainsLossesonPeriodicSettlementsofInterestRateSwaps" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Realized Gains (Losses) on Periodic Settlements of Interest Rate Swaps</link:label>
    <link:label id="lab_efc_NetRealizedGainsLossesonPeriodicSettlementsofInterestRateSwaps_label_en-US" xlink:label="lab_efc_NetRealizedGainsLossesonPeriodicSettlementsofInterestRateSwaps" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Realized Gains (Losses) on Periodic Settlements of Interest Rate Swaps</link:label>
    <link:label id="lab_efc_NetRealizedGainsLossesonPeriodicSettlementsofInterestRateSwaps_documentation_en-US" xlink:label="lab_efc_NetRealizedGainsLossesonPeriodicSettlementsofInterestRateSwaps" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Realized Gains (Losses) on Periodic Settlements of Interest Rate Swaps</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NetRealizedGainsLossesonPeriodicSettlementsofInterestRateSwaps" xlink:href="efc-20201231.xsd#efc_NetRealizedGainsLossesonPeriodicSettlementsofInterestRateSwaps"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NetRealizedGainsLossesonPeriodicSettlementsofInterestRateSwaps" xlink:to="lab_efc_NetRealizedGainsLossesonPeriodicSettlementsofInterestRateSwaps" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsNonvestedNumberOfShares_c35cbdef-1c0b-427f-8ea5-4377889a68d4_terseLabel_en-US" xlink:label="lab_us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsNonvestedNumberOfShares" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">OP LTIP Units Unvested and Outstanding end of period</link:label>
    <link:label id="lab_us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsNonvestedNumberOfShares_label_en-US" xlink:label="lab_us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsNonvestedNumberOfShares" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Compensation Arrangement by Share-based Payment Award, Options, Nonvested, Number of Shares</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsNonvestedNumberOfShares" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsNonvestedNumberOfShares"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsNonvestedNumberOfShares" xlink:to="lab_us-gaap_SharebasedCompensationArrangementBySharebasedPaymentAwardOptionsNonvestedNumberOfShares" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FixedRateResidentialMortgages15YearMember_dd35b893-2e22-493b-8388-ef56138c80cc_terseLabel_en-US" xlink:label="lab_efc_FixedRateResidentialMortgages15YearMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">15-year fixed-rate mortgages</link:label>
    <link:label id="lab_efc_FixedRateResidentialMortgages15YearMember_label_en-US" xlink:label="lab_efc_FixedRateResidentialMortgages15YearMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed Rate Residential Mortgages, 15 Year [Member]</link:label>
    <link:label id="lab_efc_FixedRateResidentialMortgages15YearMember_documentation_en-US" xlink:label="lab_efc_FixedRateResidentialMortgages15YearMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fixed Rate Residential Mortgages, 15 Year [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FixedRateResidentialMortgages15YearMember" xlink:href="efc-20201231.xsd#efc_FixedRateResidentialMortgages15YearMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FixedRateResidentialMortgages15YearMember" xlink:to="lab_efc_FixedRateResidentialMortgages15YearMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InterestOnlyPrivateLabelSecuritiesMember_98bada44-d26d-460c-8507-24e357f35713_terseLabel_en-US" xlink:label="lab_efc_InterestOnlyPrivateLabelSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Only Private Label Securities [Member]</link:label>
    <link:label id="lab_efc_InterestOnlyPrivateLabelSecuritiesMember_label_en-US" xlink:label="lab_efc_InterestOnlyPrivateLabelSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Only Private Label Securities [Member]</link:label>
    <link:label id="lab_efc_InterestOnlyPrivateLabelSecuritiesMember_documentation_en-US" xlink:label="lab_efc_InterestOnlyPrivateLabelSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest only private label securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InterestOnlyPrivateLabelSecuritiesMember" xlink:href="efc-20201231.xsd#efc_InterestOnlyPrivateLabelSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InterestOnlyPrivateLabelSecuritiesMember" xlink:to="lab_efc_InterestOnlyPrivateLabelSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DedicatedorpartiallydedicatedpersonnelMember_9071fd64-065a-427b-9b03-087ce708d8da_terseLabel_en-US" xlink:label="lab_efc_DedicatedorpartiallydedicatedpersonnelMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dedicated or partially dedicated personnel [Member]</link:label>
    <link:label id="lab_efc_DedicatedorpartiallydedicatedpersonnelMember_label_en-US" xlink:label="lab_efc_DedicatedorpartiallydedicatedpersonnelMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dedicated or partially dedicated personnel [Member]</link:label>
    <link:label id="lab_efc_DedicatedorpartiallydedicatedpersonnelMember_documentation_en-US" xlink:label="lab_efc_DedicatedorpartiallydedicatedpersonnelMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dedicated or partially dedicated personnel [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DedicatedorpartiallydedicatedpersonnelMember" xlink:href="efc-20201231.xsd#efc_DedicatedorpartiallydedicatedpersonnelMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DedicatedorpartiallydedicatedpersonnelMember" xlink:to="lab_efc_DedicatedorpartiallydedicatedpersonnelMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtInstrumentMeasurementInput_d2c01747-44a5-49ec-ad6c-cd3336f93187_terseLabel_en-US" xlink:label="lab_us-gaap_DebtInstrumentMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt instrument, measurement input</link:label>
    <link:label id="lab_us-gaap_DebtInstrumentMeasurementInput_label_en-US" xlink:label="lab_us-gaap_DebtInstrumentMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Instrument, Measurement Input</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtInstrumentMeasurementInput" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtInstrumentMeasurementInput"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtInstrumentMeasurementInput" xlink:to="lab_us-gaap_DebtInstrumentMeasurementInput" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_USBankN.A.Member_2a77db46-0d04-4c94-8d7f-2cf0ccbd19e0_terseLabel_en-US" xlink:label="lab_efc_USBankN.A.Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">US Bank N.A. [Member]</link:label>
    <link:label id="lab_efc_USBankN.A.Member_label_en-US" xlink:label="lab_efc_USBankN.A.Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">US Bank N.A. [Member]</link:label>
    <link:label id="lab_efc_USBankN.A.Member_documentation_en-US" xlink:label="lab_efc_USBankN.A.Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">US Bank N.A. [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_USBankN.A.Member" xlink:href="efc-20201231.xsd#efc_USBankN.A.Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_USBankN.A.Member" xlink:to="lab_efc_USBankN.A.Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentHoldingsOtherThanSecuritiesLineItems_9e08e7b7-1d04-445f-8d23-6599c6e6f26e_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentHoldingsOtherThanSecuritiesLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Holdings, Other than Securities [Line Items]</link:label>
    <link:label id="lab_us-gaap_InvestmentHoldingsOtherThanSecuritiesLineItems_label_en-US" xlink:label="lab_us-gaap_InvestmentHoldingsOtherThanSecuritiesLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Holdings, Other than Securities [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentHoldingsOtherThanSecuritiesLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentHoldingsOtherThanSecuritiesLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentHoldingsOtherThanSecuritiesLineItems" xlink:to="lab_us-gaap_InvestmentHoldingsOtherThanSecuritiesLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year3Point00PercentMember_d3427305-21b0-4244-a6a8-d503eaf67c55_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year3Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 3 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year3Point00PercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year3Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 3 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year3Point00PercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year3Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 3 Point 00 Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociation30Year3Point00PercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociation30Year3Point00PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociation30Year3Point00PercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociation30Year3Point00PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DerivativesCost_9b155c30-922a-45fa-87b0-9250760f3869_terseLabel_en-US" xlink:label="lab_efc_DerivativesCost" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial derivative-assets, at fair value, Net Cost</link:label>
    <link:label id="lab_efc_DerivativesCost_label_en-US" xlink:label="lab_efc_DerivativesCost" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivatives Cost</link:label>
    <link:label id="lab_efc_DerivativesCost_documentation_en-US" xlink:label="lab_efc_DerivativesCost" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Represents cost basis of derivatives carried at fair value.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DerivativesCost" xlink:href="efc-20201231.xsd#efc_DerivativesCost"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DerivativesCost" xlink:to="lab_efc_DerivativesCost" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MarketableSecuritiesPolicy_744938f6-ade9-42f3-88d4-49dc672ea928_terseLabel_en-US" xlink:label="lab_us-gaap_MarketableSecuritiesPolicy" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accounting for Securities</link:label>
    <link:label id="lab_us-gaap_MarketableSecuritiesPolicy_label_en-US" xlink:label="lab_us-gaap_MarketableSecuritiesPolicy" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Marketable Securities, Policy [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MarketableSecuritiesPolicy" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MarketableSecuritiesPolicy"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MarketableSecuritiesPolicy" xlink:to="lab_us-gaap_MarketableSecuritiesPolicy" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProceedsFromDerivativeInstrumentInvestingActivities_f4e4cb84-3920-4157-9239-51e7127b7780_terseLabel_en-US" xlink:label="lab_us-gaap_ProceedsFromDerivativeInstrumentInvestingActivities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from financial derivatives</link:label>
    <link:label id="lab_us-gaap_ProceedsFromDerivativeInstrumentInvestingActivities_label_en-US" xlink:label="lab_us-gaap_ProceedsFromDerivativeInstrumentInvestingActivities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from Derivative Instrument, Investing Activities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProceedsFromDerivativeInstrumentInvestingActivities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromDerivativeInstrumentInvestingActivities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProceedsFromDerivativeInstrumentInvestingActivities" xlink:to="lab_us-gaap_ProceedsFromDerivativeInstrumentInvestingActivities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NetIncreaseDecreaseInEquityFromTransactions_3a362bf8-4e94-4def-be52-fccb356c37b8_terseLabel_en-US" xlink:label="lab_efc_NetIncreaseDecreaseInEquityFromTransactions" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net increase (decrease) in equity from transactions</link:label>
    <link:label id="lab_efc_NetIncreaseDecreaseInEquityFromTransactions_label_en-US" xlink:label="lab_efc_NetIncreaseDecreaseInEquityFromTransactions" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net increase (decrease) in equity from transactions</link:label>
    <link:label id="lab_efc_NetIncreaseDecreaseInEquityFromTransactions_documentation_en-US" xlink:label="lab_efc_NetIncreaseDecreaseInEquityFromTransactions" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Increase (Decrease) In Equity From Transactions</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NetIncreaseDecreaseInEquityFromTransactions" xlink:href="efc-20201231.xsd#efc_NetIncreaseDecreaseInEquityFromTransactions"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NetIncreaseDecreaseInEquityFromTransactions" xlink:to="lab_efc_NetIncreaseDecreaseInEquityFromTransactions" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProspectiveAdoptionOfNewAccountingPronouncementsAxis_1bf6bb7a-558e-42ef-b994-549a953614d5_terseLabel_en-US" xlink:label="lab_us-gaap_ProspectiveAdoptionOfNewAccountingPronouncementsAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Prospective Adoption of New Accounting Pronouncements [Axis]</link:label>
    <link:label id="lab_us-gaap_ProspectiveAdoptionOfNewAccountingPronouncementsAxis_label_en-US" xlink:label="lab_us-gaap_ProspectiveAdoptionOfNewAccountingPronouncementsAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Prospective Adoption of New Accounting Pronouncements [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProspectiveAdoptionOfNewAccountingPronouncementsAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProspectiveAdoptionOfNewAccountingPronouncementsAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProspectiveAdoptionOfNewAccountingPronouncementsAxis" xlink:to="lab_us-gaap_ProspectiveAdoptionOfNewAccountingPronouncementsAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MortgageBackedSecuritiesMember_85a3ea42-0085-466f-af7a-f3c03c5622d3_terseLabel_en-US" xlink:label="lab_us-gaap_MortgageBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Collateralized Mortgage Backed Securities [Member]</link:label>
    <link:label id="lab_us-gaap_MortgageBackedSecuritiesMember_label_en-US" xlink:label="lab_us-gaap_MortgageBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Collateralized Mortgage Backed Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MortgageBackedSecuritiesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MortgageBackedSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MortgageBackedSecuritiesMember" xlink:to="lab_us-gaap_MortgageBackedSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlements_e3c18dc8-13ea-4245-9061-ee435470cfa1_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlements" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities, Payments</link:label>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlements_label_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlements" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement with Unobservable Inputs Reconciliation, Recurring Basis, Liability, Settlements</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlements" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlements"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlements" xlink:to="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisLiabilitySettlements" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ConcentrationRiskReverseRepurchaseAgreements_22c7de8a-cc4d-40ab-be89-f9205c02c955_terseLabel_en-US" xlink:label="lab_efc_ConcentrationRiskReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk, Reverse Repurchase Agreements</link:label>
    <link:label id="lab_efc_ConcentrationRiskReverseRepurchaseAgreements_label_en-US" xlink:label="lab_efc_ConcentrationRiskReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk, Reverse Repurchase Agreements</link:label>
    <link:label id="lab_efc_ConcentrationRiskReverseRepurchaseAgreements_documentation_en-US" xlink:label="lab_efc_ConcentrationRiskReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Aggregate percentage of reverse repurchase agreements held with counterparties with percentages of risk above a specified threshold.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ConcentrationRiskReverseRepurchaseAgreements" xlink:href="efc-20201231.xsd#efc_ConcentrationRiskReverseRepurchaseAgreements"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ConcentrationRiskReverseRepurchaseAgreements" xlink:to="lab_efc_ConcentrationRiskReverseRepurchaseAgreements" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2021Member_3b1c3118-3ae1-4a83-8097-578446acd24d_terseLabel_en-US" xlink:label="lab_efc_A2021Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2021 [Member]</link:label>
    <link:label id="lab_efc_A2021Member_label_en-US" xlink:label="lab_efc_A2021Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2021 [Member]</link:label>
    <link:label id="lab_efc_A2021Member_documentation_en-US" xlink:label="lab_efc_A2021Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2021 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2021Member" xlink:href="efc-20201231.xsd#efc_A2021Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2021Member" xlink:to="lab_efc_A2021Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentsDebtAndEquitySecuritiesAbstract_0ebeff19-1859-4ff3-9aaa-2e375d29b83d_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentsDebtAndEquitySecuritiesAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments, Debt and Equity Securities [Abstract]</link:label>
    <link:label id="lab_us-gaap_InvestmentsDebtAndEquitySecuritiesAbstract_label_en-US" xlink:label="lab_us-gaap_InvestmentsDebtAndEquitySecuritiesAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments, Debt and Equity Securities [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentsDebtAndEquitySecuritiesAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentsDebtAndEquitySecuritiesAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentsDebtAndEquitySecuritiesAbstract" xlink:to="lab_us-gaap_InvestmentsDebtAndEquitySecuritiesAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EarningsPerShareDilutedDistributed_b25256b2-4077-4371-bc07-1098f4b374a9_terseLabel_en-US" xlink:label="lab_us-gaap_EarningsPerShareDilutedDistributed" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Distributed (in usd per share)</link:label>
    <link:label id="lab_us-gaap_EarningsPerShareDilutedDistributed_label_en-US" xlink:label="lab_us-gaap_EarningsPerShareDilutedDistributed" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share, Diluted, Distributed</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EarningsPerShareDilutedDistributed" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EarningsPerShareDilutedDistributed"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EarningsPerShareDilutedDistributed" xlink:to="lab_us-gaap_EarningsPerShareDilutedDistributed" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NetRealizedGainsLossesonPeriodicSettlements_d25fb8ad-2592-41ef-a5a6-e619e41d99a3_terseLabel_en-US" xlink:label="lab_efc_NetRealizedGainsLossesonPeriodicSettlements" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Realized Gains (Losses) Other Than Periodic Settlements of Interest Rate Swaps</link:label>
    <link:label id="lab_efc_NetRealizedGainsLossesonPeriodicSettlements_label_en-US" xlink:label="lab_efc_NetRealizedGainsLossesonPeriodicSettlements" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Realized Gains (Losses) on Periodic Settlements</link:label>
    <link:label id="lab_efc_NetRealizedGainsLossesonPeriodicSettlements_documentation_en-US" xlink:label="lab_efc_NetRealizedGainsLossesonPeriodicSettlements" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Realized Gains (Losses) on Periodic Settlements</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NetRealizedGainsLossesonPeriodicSettlements" xlink:href="efc-20201231.xsd#efc_NetRealizedGainsLossesonPeriodicSettlements"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NetRealizedGainsLossesonPeriodicSettlements" xlink:to="lab_efc_NetRealizedGainsLossesonPeriodicSettlements" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_IncentiveFeeHurdleRateFixed_ecdd3cba-29c4-453f-a289-89c76a68fa1f_terseLabel_en-US" xlink:label="lab_efc_IncentiveFeeHurdleRateFixed" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Incentive fee hurdle rate fixed</link:label>
    <link:label id="lab_efc_IncentiveFeeHurdleRateFixed_label_en-US" xlink:label="lab_efc_IncentiveFeeHurdleRateFixed" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Incentive fee hurdle rate fixed</link:label>
    <link:label id="lab_efc_IncentiveFeeHurdleRateFixed_documentation_en-US" xlink:label="lab_efc_IncentiveFeeHurdleRateFixed" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The fixed hurdle rate used in the incentive fee calculation per the management agreement.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_IncentiveFeeHurdleRateFixed" xlink:href="efc-20201231.xsd#efc_IncentiveFeeHurdleRateFixed"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_IncentiveFeeHurdleRateFixed" xlink:to="lab_efc_IncentiveFeeHurdleRateFixed" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityRealizedGainLoss_800ac642-ffc1-48fa-ace2-d516e4845c83_terseLabel_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityRealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities, Realized Gain/(Loss)</link:label>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityRealizedGainLoss_label_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityRealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement With Unobservable Inputs Reconciliations, Recurring Basis, Liability, Realized Gain/(Loss)</link:label>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityRealizedGainLoss_documentation_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityRealizedGainLoss" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement With Unobservable Inputs Reconciliations, Recurring Basis, Liability, Realized Gain/(Loss)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityRealizedGainLoss" xlink:href="efc-20201231.xsd#efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityRealizedGainLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityRealizedGainLoss" xlink:to="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityRealizedGainLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMethodInvestmentMeasurementInput_31943c36-cff4-4113-bd7c-7f1f9bc5b201_terseLabel_en-US" xlink:label="lab_efc_EquityMethodInvestmentMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity method investment, measurement input</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentMeasurementInput_label_en-US" xlink:label="lab_efc_EquityMethodInvestmentMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Measurement Input</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentMeasurementInput_documentation_en-US" xlink:label="lab_efc_EquityMethodInvestmentMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Measurement Input</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMethodInvestmentMeasurementInput" xlink:href="efc-20201231.xsd#efc_EquityMethodInvestmentMeasurementInput"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMethodInvestmentMeasurementInput" xlink:to="lab_efc_EquityMethodInvestmentMeasurementInput" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OffsettingOfAssetsAndLiabilitiesTableTextBlock_b071cfe3-7399-4687-a6a6-b1ac6a4ba873_terseLabel_en-US" xlink:label="lab_efc_OffsettingOfAssetsAndLiabilitiesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Offsetting of Assets and Liabilities</link:label>
    <link:label id="lab_efc_OffsettingOfAssetsAndLiabilitiesTableTextBlock_label_en-US" xlink:label="lab_efc_OffsettingOfAssetsAndLiabilitiesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Offsetting of Assets and Liabilities [Table Text Block]</link:label>
    <link:label id="lab_efc_OffsettingOfAssetsAndLiabilitiesTableTextBlock_documentation_en-US" xlink:label="lab_efc_OffsettingOfAssetsAndLiabilitiesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Tabular disclosure of derivatives, repurchase agreements, and reverse repurchase agreements that are subject to offsetting.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OffsettingOfAssetsAndLiabilitiesTableTextBlock" xlink:href="efc-20201231.xsd#efc_OffsettingOfAssetsAndLiabilitiesTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OffsettingOfAssetsAndLiabilitiesTableTextBlock" xlink:to="lab_efc_OffsettingOfAssetsAndLiabilitiesTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Drawndownamountbeingguaranteed_04357df5-09ab-4def-b61d-5373acf532ae_terseLabel_en-US" xlink:label="lab_efc_Drawndownamountbeingguaranteed" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Drawn down amount being guaranteed</link:label>
    <link:label id="lab_efc_Drawndownamountbeingguaranteed_label_en-US" xlink:label="lab_efc_Drawndownamountbeingguaranteed" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Drawn down amount being guaranteed</link:label>
    <link:label id="lab_efc_Drawndownamountbeingguaranteed_documentation_en-US" xlink:label="lab_efc_Drawndownamountbeingguaranteed" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount of borrowings outstanding under related party's MRA for which the Company has guaranteed repayment.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Drawndownamountbeingguaranteed" xlink:href="efc-20201231.xsd#efc_Drawndownamountbeingguaranteed"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Drawndownamountbeingguaranteed" xlink:to="lab_efc_Drawndownamountbeingguaranteed" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LongMember_77fa869c-60e0-41d3-b854-de8ec146b950_terseLabel_en-US" xlink:label="lab_us-gaap_LongMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long</link:label>
    <link:label id="lab_us-gaap_LongMember_label_en-US" xlink:label="lab_us-gaap_LongMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LongMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LongMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LongMember" xlink:to="lab_us-gaap_LongMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsOtherFourPointFivePercentMember_9079de01-eb38-412d-917b-87471f4f8091_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsOtherFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Other) Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsOtherFourPointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsOtherFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Other) Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsOtherFourPointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsOtherFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Other) Four Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsOtherFourPointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsOtherFourPointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsOtherFourPointFivePercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsOtherFourPointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_RangeMember_be647dce-210a-4202-b601-03d01d86b1f7_terseLabel_en-US" xlink:label="lab_srt_RangeMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Statistical Measurement [Domain]</link:label>
    <link:label id="lab_srt_RangeMember_label_en-US" xlink:label="lab_srt_RangeMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Statistical Measurement [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_RangeMember" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_RangeMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_RangeMember" xlink:to="lab_srt_RangeMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePercentMember_8ac4b159-e791-40ef-b002-f0862ab57ac2_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentOwnedAtFairValue_10de9544-0db1-45dc-b23d-cc50c576f634_verboseLabel_en-US" xlink:label="lab_us-gaap_InvestmentOwnedAtFairValue" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments, at fair value (Cost - $2,970,306)</link:label>
    <link:label id="lab_us-gaap_InvestmentOwnedAtFairValue_22a66357-30d2-4ae0-a256-cf1f9f8dec55_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentOwnedAtFairValue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value</link:label>
    <link:label id="lab_us-gaap_InvestmentOwnedAtFairValue_label_en-US" xlink:label="lab_us-gaap_InvestmentOwnedAtFairValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Owned, at Fair Value</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentOwnedAtFairValue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentOwnedAtFairValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentOwnedAtFairValue" xlink:to="lab_us-gaap_InvestmentOwnedAtFairValue" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardNonOptionEquityInstrumentsOutstandingRollForward_178d5d7b-8a65-420c-be2c-9c0cfaf23861_terseLabel_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardNonOptionEquityInstrumentsOutstandingRollForward" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Compensation Arrangement by Share-based Payment Award, Non-Option Equity Instruments, Outstanding [Roll Forward]</link:label>
    <link:label id="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardNonOptionEquityInstrumentsOutstandingRollForward_label_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardNonOptionEquityInstrumentsOutstandingRollForward" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Compensation Arrangement by Share-based Payment Award, Non-Option Equity Instruments, Outstanding [Roll Forward]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardNonOptionEquityInstrumentsOutstandingRollForward" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardNonOptionEquityInstrumentsOutstandingRollForward"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardNonOptionEquityInstrumentsOutstandingRollForward" xlink:to="lab_us-gaap_ShareBasedCompensationArrangementByShareBasedPaymentAwardNonOptionEquityInstrumentsOutstandingRollForward" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_VariousMoneyMarketFundsMember_6e65c832-7c45-4b3d-929a-c28aa749a3cd_terseLabel_en-US" xlink:label="lab_efc_VariousMoneyMarketFundsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Various Money Market Funds [Member]</link:label>
    <link:label id="lab_efc_VariousMoneyMarketFundsMember_label_en-US" xlink:label="lab_efc_VariousMoneyMarketFundsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Various Money Market Funds [Member]</link:label>
    <link:label id="lab_efc_VariousMoneyMarketFundsMember_documentation_en-US" xlink:label="lab_efc_VariousMoneyMarketFundsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Various Money Market Funds [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_VariousMoneyMarketFundsMember" xlink:href="efc-20201231.xsd#efc_VariousMoneyMarketFundsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_VariousMoneyMarketFundsMember" xlink:to="lab_efc_VariousMoneyMarketFundsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityDomain_f2d7c7ee-3611-43a1-8ee7-8093b10bc52d_terseLabel_en-US" xlink:label="lab_dei_EntityDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity [Domain]</link:label>
    <link:label id="lab_dei_EntityDomain_label_en-US" xlink:label="lab_dei_EntityDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityDomain" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityDomain" xlink:to="lab_dei_EntityDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RealizedLossesWriteoffsonloans_5be269d9-de58-4581-947d-e7782d68af75_terseLabel_en-US" xlink:label="lab_efc_RealizedLossesWriteoffsonloans" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized Losses, Write offs on loans</link:label>
    <link:label id="lab_efc_RealizedLossesWriteoffsonloans_label_en-US" xlink:label="lab_efc_RealizedLossesWriteoffsonloans" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized Losses, Write offs on loans</link:label>
    <link:label id="lab_efc_RealizedLossesWriteoffsonloans_documentation_en-US" xlink:label="lab_efc_RealizedLossesWriteoffsonloans" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized losses resulting from a complete or partial write off once the Company has determined all or a portion of an impaired loan's cost basis is uncollectible.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RealizedLossesWriteoffsonloans" xlink:href="efc-20201231.xsd#efc_RealizedLossesWriteoffsonloans"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RealizedLossesWriteoffsonloans" xlink:to="lab_efc_RealizedLossesWriteoffsonloans" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CommercialMortgageLoansMember_8fe65ce5-49f8-4348-9022-ca34c05b1159_terseLabel_en-US" xlink:label="lab_efc_CommercialMortgageLoansMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commercial mortgage loans</link:label>
    <link:label id="lab_efc_CommercialMortgageLoansMember_label_en-US" xlink:label="lab_efc_CommercialMortgageLoansMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commercial Mortgage Loans [Member]</link:label>
    <link:label id="lab_efc_CommercialMortgageLoansMember_documentation_en-US" xlink:label="lab_efc_CommercialMortgageLoansMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commercial Mortgage Loans [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CommercialMortgageLoansMember" xlink:href="efc-20201231.xsd#efc_CommercialMortgageLoansMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CommercialMortgageLoansMember" xlink:to="lab_efc_CommercialMortgageLoansMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DueFromBrokerMember_96e66391-cf9e-4fac-9081-f8a0e8be3b05_terseLabel_en-US" xlink:label="lab_efc_DueFromBrokerMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Due From Broker [Member]</link:label>
    <link:label id="lab_efc_DueFromBrokerMember_label_en-US" xlink:label="lab_efc_DueFromBrokerMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Due From Broker [Member]</link:label>
    <link:label id="lab_efc_DueFromBrokerMember_documentation_en-US" xlink:label="lab_efc_DueFromBrokerMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Due From Broker [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DueFromBrokerMember" xlink:href="efc-20201231.xsd#efc_DueFromBrokerMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DueFromBrokerMember" xlink:to="lab_efc_DueFromBrokerMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock_7ee205ba-2c00-446b-bebf-cbb4fe4c2099_verboseLabel_en-US" xlink:label="lab_us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-Term Incentive Plan Units</link:label>
    <link:label id="lab_us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock_a9f17f7f-019f-45e1-a935-a630f1e35309_terseLabel_en-US" xlink:label="lab_us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Payment Arrangement</link:label>
    <link:label id="lab_us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock_label_en-US" xlink:label="lab_us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Payment Arrangement [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock" xlink:to="lab_us-gaap_DisclosureOfCompensationRelatedCostsShareBasedPaymentsTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Proceedsfromprincipalpaymentsofinvestmentsnoncash_b3552cbe-3aed-4343-b273-0032ab1684f2_terseLabel_en-US" xlink:label="lab_efc_Proceedsfromprincipalpaymentsofinvestmentsnoncash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from principal payments of investments (non-cash)</link:label>
    <link:label id="lab_efc_Proceedsfromprincipalpaymentsofinvestmentsnoncash_label_en-US" xlink:label="lab_efc_Proceedsfromprincipalpaymentsofinvestmentsnoncash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from principal payments of investments (non-cash)</link:label>
    <link:label id="lab_efc_Proceedsfromprincipalpaymentsofinvestmentsnoncash_documentation_en-US" xlink:label="lab_efc_Proceedsfromprincipalpaymentsofinvestmentsnoncash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-cash activity associated with the maturity and collection of all investments such as debt during the period.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Proceedsfromprincipalpaymentsofinvestmentsnoncash" xlink:href="efc-20201231.xsd#efc_Proceedsfromprincipalpaymentsofinvestmentsnoncash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Proceedsfromprincipalpaymentsofinvestmentsnoncash" xlink:to="lab_efc_Proceedsfromprincipalpaymentsofinvestmentsnoncash" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NoncontrollingInterestinMortgagerelatedPrivatePartnershipMember_af225234-88ff-4dc1-98b9-a6ef8be65a23_terseLabel_en-US" xlink:label="lab_efc_NoncontrollingInterestinMortgagerelatedPrivatePartnershipMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling Interest in Mortgage-related Private Partnership</link:label>
    <link:label id="lab_efc_NoncontrollingInterestinMortgagerelatedPrivatePartnershipMember_label_en-US" xlink:label="lab_efc_NoncontrollingInterestinMortgagerelatedPrivatePartnershipMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling Interest in Mortgage-related Private Partnership [Member]</link:label>
    <link:label id="lab_efc_NoncontrollingInterestinMortgagerelatedPrivatePartnershipMember_documentation_en-US" xlink:label="lab_efc_NoncontrollingInterestinMortgagerelatedPrivatePartnershipMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling Interest in Mortgage-related Private Partnership [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NoncontrollingInterestinMortgagerelatedPrivatePartnershipMember" xlink:href="efc-20201231.xsd#efc_NoncontrollingInterestinMortgagerelatedPrivatePartnershipMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NoncontrollingInterestinMortgagerelatedPrivatePartnershipMember" xlink:to="lab_efc_NoncontrollingInterestinMortgagerelatedPrivatePartnershipMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SubsidiarySaleOfStockLineItems_84e558ab-c514-4c95-8a9e-790fe2c0f971_terseLabel_en-US" xlink:label="lab_us-gaap_SubsidiarySaleOfStockLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsidiary, Sale of Stock [Line Items]</link:label>
    <link:label id="lab_us-gaap_SubsidiarySaleOfStockLineItems_label_en-US" xlink:label="lab_us-gaap_SubsidiarySaleOfStockLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsidiary, Sale of Stock [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SubsidiarySaleOfStockLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SubsidiarySaleOfStockLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SubsidiarySaleOfStockLineItems" xlink:to="lab_us-gaap_SubsidiarySaleOfStockLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CorporateDebtandnonexchangetradedcorporateequityMember_4d9df746-74cf-4139-82cf-6b7d30e77e8b_terseLabel_en-US" xlink:label="lab_efc_CorporateDebtandnonexchangetradedcorporateequityMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate Debt and non-exchange traded corporate equity [Member]</link:label>
    <link:label id="lab_efc_CorporateDebtandnonexchangetradedcorporateequityMember_label_en-US" xlink:label="lab_efc_CorporateDebtandnonexchangetradedcorporateequityMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate Debt and non-exchange traded corporate equity [Member]</link:label>
    <link:label id="lab_efc_CorporateDebtandnonexchangetradedcorporateequityMember_documentation_en-US" xlink:label="lab_efc_CorporateDebtandnonexchangetradedcorporateequityMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate Debt and non-exchange traded corporate equity [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CorporateDebtandnonexchangetradedcorporateequityMember" xlink:href="efc-20201231.xsd#efc_CorporateDebtandnonexchangetradedcorporateequityMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CorporateDebtandnonexchangetradedcorporateequityMember" xlink:to="lab_efc_CorporateDebtandnonexchangetradedcorporateequityMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InterestOnlyFloatingRateAgencySecuritiesMember_3b48be47-0430-4547-af20-d2715c6ac9dd_terseLabel_en-US" xlink:label="lab_efc_InterestOnlyFloatingRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Only Floating Rate Agency Securities [Member]</link:label>
    <link:label id="lab_efc_InterestOnlyFloatingRateAgencySecuritiesMember_label_en-US" xlink:label="lab_efc_InterestOnlyFloatingRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Only Floating Rate Agency Securities [Member]</link:label>
    <link:label id="lab_efc_InterestOnlyFloatingRateAgencySecuritiesMember_documentation_en-US" xlink:label="lab_efc_InterestOnlyFloatingRateAgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">A security issued by a government agency or government sponsored enterprise which makes variable interest rate payments.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InterestOnlyFloatingRateAgencySecuritiesMember" xlink:href="efc-20201231.xsd#efc_InterestOnlyFloatingRateAgencySecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InterestOnlyFloatingRateAgencySecuritiesMember" xlink:to="lab_efc_InterestOnlyFloatingRateAgencySecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InvestmentsAndFinancialDerivativesAbstract_6c938bc0-0b2e-4d3b-ac02-1a27eab8971c_terseLabel_en-US" xlink:label="lab_efc_InvestmentsAndFinancialDerivativesAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments and Financial Derivatives [Abstract]</link:label>
    <link:label id="lab_efc_InvestmentsAndFinancialDerivativesAbstract_label_en-US" xlink:label="lab_efc_InvestmentsAndFinancialDerivativesAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments and Financial Derivatives [Abstract]</link:label>
    <link:label id="lab_efc_InvestmentsAndFinancialDerivativesAbstract_documentation_en-US" xlink:label="lab_efc_InvestmentsAndFinancialDerivativesAbstract" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments and Financial Derivatives</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InvestmentsAndFinancialDerivativesAbstract" xlink:href="efc-20201231.xsd#efc_InvestmentsAndFinancialDerivativesAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InvestmentsAndFinancialDerivativesAbstract" xlink:to="lab_efc_InvestmentsAndFinancialDerivativesAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonManagerMember_0129da8d-8e23-44f3-960c-72b7a0ff6e47_verboseLabel_en-US" xlink:label="lab_efc_NonManagerMember" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Manager [Member]</link:label>
    <link:label id="lab_efc_NonManagerMember_label_en-US" xlink:label="lab_efc_NonManagerMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non Manager [Member]</link:label>
    <link:label id="lab_efc_NonManagerMember_documentation_en-US" xlink:label="lab_efc_NonManagerMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non Manager [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonManagerMember" xlink:href="efc-20201231.xsd#efc_NonManagerMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonManagerMember" xlink:to="lab_efc_NonManagerMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_TechnologySectorMember_00c731f9-d928-4f65-a83c-bfe2568f008a_terseLabel_en-US" xlink:label="lab_us-gaap_TechnologySectorMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Technology Sector [Member]</link:label>
    <link:label id="lab_us-gaap_TechnologySectorMember_label_en-US" xlink:label="lab_us-gaap_TechnologySectorMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Technology Sector [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_TechnologySectorMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_TechnologySectorMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_TechnologySectorMember" xlink:to="lab_us-gaap_TechnologySectorMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentInterestRate_eac26e3f-a9a0-443f-8054-66ef6fd79e1c_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentInterestRate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Interest Rate</link:label>
    <link:label id="lab_us-gaap_InvestmentInterestRate_bfc58aaf-9bd3-4820-a2fc-0ed55b5e6fd4_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_InvestmentInterestRate" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Interest Rate</link:label>
    <link:label id="lab_us-gaap_InvestmentInterestRate_67f4a35a-f5f2-4cc7-b5e9-2ec06b67d8ae_verboseLabel_en-US" xlink:label="lab_us-gaap_InvestmentInterestRate" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Weighted Average Coupon</link:label>
    <link:label id="lab_us-gaap_InvestmentInterestRate_label_en-US" xlink:label="lab_us-gaap_InvestmentInterestRate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Interest Rate</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentInterestRate" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentInterestRate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentInterestRate" xlink:to="lab_us-gaap_InvestmentInterestRate" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OtherCommitmentsTable_0bd41dae-77ed-48a4-8ac2-bb6b41fcdb17_terseLabel_en-US" xlink:label="lab_us-gaap_OtherCommitmentsTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Commitments [Table]</link:label>
    <link:label id="lab_us-gaap_OtherCommitmentsTable_label_en-US" xlink:label="lab_us-gaap_OtherCommitmentsTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Commitments [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OtherCommitmentsTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OtherCommitmentsTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OtherCommitmentsTable" xlink:to="lab_us-gaap_OtherCommitmentsTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OperatingPartnershipLTIPUnitsIssued_e08bb621-39db-4fff-be4b-2213959039dc_terseLabel_en-US" xlink:label="lab_efc_OperatingPartnershipLTIPUnitsIssued" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership LTIP Units Issued (in units)</link:label>
    <link:label id="lab_efc_OperatingPartnershipLTIPUnitsIssued_label_en-US" xlink:label="lab_efc_OperatingPartnershipLTIPUnitsIssued" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership LTIP Units Issued</link:label>
    <link:label id="lab_efc_OperatingPartnershipLTIPUnitsIssued_documentation_en-US" xlink:label="lab_efc_OperatingPartnershipLTIPUnitsIssued" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership LTIP Units Issued</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OperatingPartnershipLTIPUnitsIssued" xlink:href="efc-20201231.xsd#efc_OperatingPartnershipLTIPUnitsIssued"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OperatingPartnershipLTIPUnitsIssued" xlink:to="lab_efc_OperatingPartnershipLTIPUnitsIssued" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_CA_15beda5a-7fed-4937-b9ed-9313422390b1_terseLabel_en-US" xlink:label="lab_stpr_CA" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CALIFORNIA</link:label>
    <link:label id="lab_stpr_CA_label_en-US" xlink:label="lab_stpr_CA" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CALIFORNIA</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_CA" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_CA"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_CA" xlink:to="lab_stpr_CA" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RetainedriskretentionrulesMember_6779b982-22cf-4d22-8b05-2d4b0a59c377_terseLabel_en-US" xlink:label="lab_efc_RetainedriskretentionrulesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Retained, risk retention rules [Member]</link:label>
    <link:label id="lab_efc_RetainedriskretentionrulesMember_label_en-US" xlink:label="lab_efc_RetainedriskretentionrulesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Retained, risk retention rules [Member]</link:label>
    <link:label id="lab_efc_RetainedriskretentionrulesMember_documentation_en-US" xlink:label="lab_efc_RetainedriskretentionrulesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Retained, risk retention rules [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RetainedriskretentionrulesMember" xlink:href="efc-20201231.xsd#efc_RetainedriskretentionrulesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RetainedriskretentionrulesMember" xlink:to="lab_efc_RetainedriskretentionrulesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OtherLiabilities_edf4130e-5168-4f86-a40b-97b034d2cb39_terseLabel_en-US" xlink:label="lab_us-gaap_OtherLiabilities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Liabilities</link:label>
    <link:label id="lab_us-gaap_OtherLiabilities_label_en-US" xlink:label="lab_us-gaap_OtherLiabilities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Liabilities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OtherLiabilities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OtherLiabilities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OtherLiabilities" xlink:to="lab_us-gaap_OtherLiabilities" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTable_15d0b31f-e18a-4223-8b41-5cafa0861297_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Restricted Cash [Table]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTable_label_en-US" xlink:label="lab_us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Restrictions on Cash and Cash Equivalents [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTable" xlink:to="lab_us-gaap_ScheduleOfRestrictedCashAndCashEquivalentsTable" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoanOnRealEstateFinalMaturityDate_eeb4e261-87ee-4b54-a420-1d6fe8fc192e_terseLabel_en-US" xlink:label="lab_srt_MortgageLoanOnRealEstateFinalMaturityDate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Final Maturity Date</link:label>
    <link:label id="lab_srt_MortgageLoanOnRealEstateFinalMaturityDate_label_en-US" xlink:label="lab_srt_MortgageLoanOnRealEstateFinalMaturityDate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Final Maturity Date</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoanOnRealEstateFinalMaturityDate" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoanOnRealEstateFinalMaturityDate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoanOnRealEstateFinalMaturityDate" xlink:to="lab_srt_MortgageLoanOnRealEstateFinalMaturityDate" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_VestDecember172022Member_bb1dd88f-52a8-4a51-8744-00ba11a21d39_terseLabel_en-US" xlink:label="lab_efc_VestDecember172022Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 17, 2022</link:label>
    <link:label id="lab_efc_VestDecember172022Member_label_en-US" xlink:label="lab_efc_VestDecember172022Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 17, 2022 [Member]</link:label>
    <link:label id="lab_efc_VestDecember172022Member_documentation_en-US" xlink:label="lab_efc_VestDecember172022Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 17, 2022</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_VestDecember172022Member" xlink:href="efc-20201231.xsd#efc_VestDecember172022Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_VestDecember172022Member" xlink:to="lab_efc_VestDecember172022Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OperatingPartnershipLTIPUnitsRedeemed_75b086a5-8ea7-41e8-8d6f-affd5e7df53d_terseLabel_en-US" xlink:label="lab_efc_OperatingPartnershipLTIPUnitsRedeemed" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership LTIP Units Redeemed (in units)</link:label>
    <link:label id="lab_efc_OperatingPartnershipLTIPUnitsRedeemed_label_en-US" xlink:label="lab_efc_OperatingPartnershipLTIPUnitsRedeemed" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership LTIP Units Redeemed</link:label>
    <link:label id="lab_efc_OperatingPartnershipLTIPUnitsRedeemed_documentation_en-US" xlink:label="lab_efc_OperatingPartnershipLTIPUnitsRedeemed" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership LTIP Units Redeemed</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OperatingPartnershipLTIPUnitsRedeemed" xlink:href="efc-20201231.xsd#efc_OperatingPartnershipLTIPUnitsRedeemed"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OperatingPartnershipLTIPUnitsRedeemed" xlink:to="lab_efc_OperatingPartnershipLTIPUnitsRedeemed" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RelatedPartyTransactionsByRelatedPartyAxis_ddece118-7d57-41aa-87c3-2ba082828a48_terseLabel_en-US" xlink:label="lab_us-gaap_RelatedPartyTransactionsByRelatedPartyAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party [Axis]</link:label>
    <link:label id="lab_us-gaap_RelatedPartyTransactionsByRelatedPartyAxis_label_en-US" xlink:label="lab_us-gaap_RelatedPartyTransactionsByRelatedPartyAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RelatedPartyTransactionsByRelatedPartyAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RelatedPartyTransactionsByRelatedPartyAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RelatedPartyTransactionsByRelatedPartyAxis" xlink:to="lab_us-gaap_RelatedPartyTransactionsByRelatedPartyAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearSixPercentMember_8834a85d-fdcd-47a1-bb38-d8db00a9c904_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearSixPercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Six Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearSixPercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearSixPercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Six Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearSixPercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearSixPercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Pools (Thirty Year) Six Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearSixPercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationPoolsThirtyYearSixPercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationPoolsThirtyYearSixPercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationPoolsThirtyYearSixPercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EstimateOfFairValueFairValueDisclosureMember_ee3ba275-5630-4704-b98f-3bbb0595b58b_terseLabel_en-US" xlink:label="lab_us-gaap_EstimateOfFairValueFairValueDisclosureMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value</link:label>
    <link:label id="lab_us-gaap_EstimateOfFairValueFairValueDisclosureMember_label_en-US" xlink:label="lab_us-gaap_EstimateOfFairValueFairValueDisclosureMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Estimate of Fair Value Measurement [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EstimateOfFairValueFairValueDisclosureMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EstimateOfFairValueFairValueDisclosureMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EstimateOfFairValueFairValueDisclosureMember" xlink:to="lab_us-gaap_EstimateOfFairValueFairValueDisclosureMember" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_CounterpartyNameAxis_4f3d7dd7-7565-4ca1-9d17-eda8e534ab27_terseLabel_en-US" xlink:label="lab_srt_CounterpartyNameAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Counterparty Name [Axis]</link:label>
    <link:label id="lab_srt_CounterpartyNameAxis_label_en-US" xlink:label="lab_srt_CounterpartyNameAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Counterparty Name [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_CounterpartyNameAxis" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_CounterpartyNameAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_CounterpartyNameAxis" xlink:to="lab_srt_CounterpartyNameAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EarningsPerShareAbstract_f97010f7-5c26-4dfb-b785-812e890cdef4_terseLabel_en-US" xlink:label="lab_us-gaap_EarningsPerShareAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share [Abstract]</link:label>
    <link:label id="lab_us-gaap_EarningsPerShareAbstract_label_en-US" xlink:label="lab_us-gaap_EarningsPerShareAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EarningsPerShareAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EarningsPerShareAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EarningsPerShareAbstract" xlink:to="lab_us-gaap_EarningsPerShareAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTableTextBlock_a5b57ab7-0f01-495c-9f28-2114e77a8a49_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Significant Unobservable Inputs, Qualitative Information</link:label>
    <link:label id="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTableTextBlock_label_en-US" xlink:label="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Measurement Inputs and Valuation Techniques [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTableTextBlock" xlink:to="lab_us-gaap_FairValueAssetsAndLiabilitiesMeasuredOnRecurringAndNonrecurringBasisValuationTechniquesTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year5Point00PercentMember_491c2f3c-b828-4f33-b8e8-cbb36d7292b2_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year5Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 5 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year5Point00PercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year5Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 5 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year5Point00PercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year5Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 5.00% [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociation30Year5Point00PercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociation30Year5Point00PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociation30Year5Point00PercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociation30Year5Point00PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AssetBackedSecuritiesMember_6a92f55b-f2ee-4f0a-890d-8b32505f4c2d_terseLabel_en-US" xlink:label="lab_us-gaap_AssetBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Asset-backed securities, backed by consumer loans</link:label>
    <link:label id="lab_us-gaap_AssetBackedSecuritiesMember_label_en-US" xlink:label="lab_us-gaap_AssetBackedSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Asset-backed Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AssetBackedSecuritiesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AssetBackedSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AssetBackedSecuritiesMember" xlink:to="lab_us-gaap_AssetBackedSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueByFairValueHierarchyLevelAxis_68f41d05-94a9-4e1a-8097-d5f320dfaa7f_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueByFairValueHierarchyLevelAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Hierarchy and NAV [Axis]</link:label>
    <link:label id="lab_us-gaap_FairValueByFairValueHierarchyLevelAxis_label_en-US" xlink:label="lab_us-gaap_FairValueByFairValueHierarchyLevelAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Hierarchy and NAV [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueByFairValueHierarchyLevelAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueByFairValueHierarchyLevelAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueByFairValueHierarchyLevelAxis" xlink:to="lab_us-gaap_FairValueByFairValueHierarchyLevelAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Proceedsfromissuanceofsharesnet_8f9db19c-ec5c-42df-8858-38c99e8c8831_terseLabel_en-US" xlink:label="lab_efc_Proceedsfromissuanceofsharesnet" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from issuance of shares, net</link:label>
    <link:label id="lab_efc_Proceedsfromissuanceofsharesnet_label_en-US" xlink:label="lab_efc_Proceedsfromissuanceofsharesnet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from issuance of shares, net</link:label>
    <link:label id="lab_efc_Proceedsfromissuanceofsharesnet_documentation_en-US" xlink:label="lab_efc_Proceedsfromissuanceofsharesnet" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net proceeds from issuance of common shares. Net proceeds after underwriters' discount and offering costs.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Proceedsfromissuanceofsharesnet" xlink:href="efc-20201231.xsd#efc_Proceedsfromissuanceofsharesnet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Proceedsfromissuanceofsharesnet" xlink:to="lab_efc_Proceedsfromissuanceofsharesnet" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MinorityInterestDecreaseFromDistributionsToNoncontrollingInterestHolders_88538c44-0e4c-4e7e-8824-0ce5b9885a0b_negatedLabel_en-US" xlink:label="lab_us-gaap_MinorityInterestDecreaseFromDistributionsToNoncontrollingInterestHolders" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Distributions to non-controlling interests</link:label>
    <link:label id="lab_us-gaap_MinorityInterestDecreaseFromDistributionsToNoncontrollingInterestHolders_label_en-US" xlink:label="lab_us-gaap_MinorityInterestDecreaseFromDistributionsToNoncontrollingInterestHolders" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noncontrolling Interest, Decrease from Distributions to Noncontrolling Interest Holders</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MinorityInterestDecreaseFromDistributionsToNoncontrollingInterestHolders" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MinorityInterestDecreaseFromDistributionsToNoncontrollingInterestHolders"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MinorityInterestDecreaseFromDistributionsToNoncontrollingInterestHolders" xlink:to="lab_us-gaap_MinorityInterestDecreaseFromDistributionsToNoncontrollingInterestHolders" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CorporateloanstransferredintoCLOsecuritization_bc25d0df-d0c2-4686-b070-d75bd6c598d6_terseLabel_en-US" xlink:label="lab_efc_CorporateloanstransferredintoCLOsecuritization" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate loans transferred into CLO securitization</link:label>
    <link:label id="lab_efc_CorporateloanstransferredintoCLOsecuritization_label_en-US" xlink:label="lab_efc_CorporateloanstransferredintoCLOsecuritization" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate loans transferred into CLO securitization</link:label>
    <link:label id="lab_efc_CorporateloanstransferredintoCLOsecuritization_documentation_en-US" xlink:label="lab_efc_CorporateloanstransferredintoCLOsecuritization" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair value of corporate loans transferred into CLO securitization</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CorporateloanstransferredintoCLOsecuritization" xlink:href="efc-20201231.xsd#efc_CorporateloanstransferredintoCLOsecuritization"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CorporateloanstransferredintoCLOsecuritization" xlink:to="lab_efc_CorporateloanstransferredintoCLOsecuritization" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageOriginatorsMember_43369c50-8be7-41d3-9eac-6b8d0e6d7287_terseLabel_en-US" xlink:label="lab_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageOriginatorsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Preferred Equity Investment in Mortgage Originators [Member]</link:label>
    <link:label id="lab_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageOriginatorsMember_label_en-US" xlink:label="lab_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageOriginatorsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Preferred Equity Investment in Mortgage Originators [Member]</link:label>
    <link:label id="lab_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageOriginatorsMember_documentation_en-US" xlink:label="lab_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageOriginatorsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Preferred Equity Investment in Mortgage Originators [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageOriginatorsMember" xlink:href="efc-20201231.xsd#efc_NonExchangeTradedPreferredEquityInvestmentinMortgageOriginatorsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageOriginatorsMember" xlink:to="lab_efc_NonExchangeTradedPreferredEquityInvestmentinMortgageOriginatorsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CLOIVSecuritizationMember_81ca7a81-4955-45f6-b0e0-6e4fd8522140_terseLabel_en-US" xlink:label="lab_efc_CLOIVSecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLO IV Securitization [Member]</link:label>
    <link:label id="lab_efc_CLOIVSecuritizationMember_label_en-US" xlink:label="lab_efc_CLOIVSecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLO IV Securitization [Member]</link:label>
    <link:label id="lab_efc_CLOIVSecuritizationMember_documentation_en-US" xlink:label="lab_efc_CLOIVSecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLO IV Securitization [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CLOIVSecuritizationMember" xlink:href="efc-20201231.xsd#efc_CLOIVSecuritizationMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CLOIVSecuritizationMember" xlink:to="lab_efc_CLOIVSecuritizationMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InterestAndOtherIncome_89c1a764-75cf-4d37-844f-92e509190298_terseLabel_en-US" xlink:label="lab_us-gaap_InterestAndOtherIncome" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total investment income</link:label>
    <link:label id="lab_us-gaap_InterestAndOtherIncome_label_en-US" xlink:label="lab_us-gaap_InterestAndOtherIncome" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest and Other Income</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InterestAndOtherIncome" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InterestAndOtherIncome"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InterestAndOtherIncome" xlink:to="lab_us-gaap_InterestAndOtherIncome" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleOfNetAssetBasedTotalReturnForShareholdersTableTextBlock_7c9fd453-7cf1-450c-88e0-9f3da89c80ca_terseLabel_en-US" xlink:label="lab_efc_ScheduleOfNetAssetBasedTotalReturnForShareholdersTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Net Asset Based Total Return For Shareholders</link:label>
    <link:label id="lab_efc_ScheduleOfNetAssetBasedTotalReturnForShareholdersTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleOfNetAssetBasedTotalReturnForShareholdersTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Net Asset Based Total Return For Shareholders [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleOfNetAssetBasedTotalReturnForShareholdersTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleOfNetAssetBasedTotalReturnForShareholdersTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Tabular disclosure for the entity's total return based on net asset value.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleOfNetAssetBasedTotalReturnForShareholdersTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleOfNetAssetBasedTotalReturnForShareholdersTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleOfNetAssetBasedTotalReturnForShareholdersTableTextBlock" xlink:to="lab_efc_ScheduleOfNetAssetBasedTotalReturnForShareholdersTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_TerminationFeeNumberOfPeriods_f069bf0d-6600-4a8f-a4db-8ab68fd6bbc5_terseLabel_en-US" xlink:label="lab_efc_TerminationFeeNumberOfPeriods" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Termination Fee, Number Of Periods</link:label>
    <link:label id="lab_efc_TerminationFeeNumberOfPeriods_label_en-US" xlink:label="lab_efc_TerminationFeeNumberOfPeriods" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Termination Fee, Number Of Periods</link:label>
    <link:label id="lab_efc_TerminationFeeNumberOfPeriods_documentation_en-US" xlink:label="lab_efc_TerminationFeeNumberOfPeriods" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Termination Fee, Number Of Periods</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_TerminationFeeNumberOfPeriods" xlink:href="efc-20201231.xsd#efc_TerminationFeeNumberOfPeriods"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_TerminationFeeNumberOfPeriods" xlink:to="lab_efc_TerminationFeeNumberOfPeriods" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_AZ_0637c2bd-755b-4909-a136-3867d29de1c1_terseLabel_en-US" xlink:label="lab_stpr_AZ" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">ARIZONA</link:label>
    <link:label id="lab_stpr_AZ_label_en-US" xlink:label="lab_stpr_AZ" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">ARIZONA</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_AZ" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_AZ"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_AZ" xlink:to="lab_stpr_AZ" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Othersecuredborrowingsatfairvalue_c7cf09f3-6667-4978-8d98-db01c247c700_terseLabel_en-US" xlink:label="lab_efc_Othersecuredborrowingsatfairvalue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other secured borrowings, at fair value</link:label>
    <link:label id="lab_efc_Othersecuredborrowingsatfairvalue_7af0fb37-db32-4948-83b1-d48659276f52_negatedTerseLabel_en-US" xlink:label="lab_efc_Othersecuredborrowingsatfairvalue" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other secured borrowings, at fair value</link:label>
    <link:label id="lab_efc_Othersecuredborrowingsatfairvalue_label_en-US" xlink:label="lab_efc_Othersecuredborrowingsatfairvalue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other secured borrowings, at fair value</link:label>
    <link:label id="lab_efc_Othersecuredborrowingsatfairvalue_documentation_en-US" xlink:label="lab_efc_Othersecuredborrowingsatfairvalue" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities, at fair value, associated with consolidated securitization trusts. Shown net of notes held by the Company (eliminated in consolidation).</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Othersecuredborrowingsatfairvalue" xlink:href="efc-20201231.xsd#efc_Othersecuredborrowingsatfairvalue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Othersecuredborrowingsatfairvalue" xlink:to="lab_efc_Othersecuredborrowingsatfairvalue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue2800U.S.TreasuryNoteCoupon2.88MaturityDate202808Member_ee33e3b4-ba96-4d10-8c94-3dfef10502ca_terseLabel_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue2800U.S.TreasuryNoteCoupon2.88MaturityDate202808Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $2,800 U.S. Treasury Note, Coupon 2.88%, Maturity Date 2028 08 [Member]</link:label>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue2800U.S.TreasuryNoteCoupon2.88MaturityDate202808Member_label_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue2800U.S.TreasuryNoteCoupon2.88MaturityDate202808Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $2,800 U.S. Treasury Note, Coupon 2.88%, Maturity Date 2028 08 [Member]</link:label>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue2800U.S.TreasuryNoteCoupon2.88MaturityDate202808Member_documentation_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue2800U.S.TreasuryNoteCoupon2.88MaturityDate202808Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $2,800 U.S. Treasury Note, Coupon 2.88%, Maturity Date 2028 08 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_JPMorganSecuritiesLLCCollateralizedbyParValue2800U.S.TreasuryNoteCoupon2.88MaturityDate202808Member" xlink:href="efc-20201231.xsd#efc_JPMorganSecuritiesLLCCollateralizedbyParValue2800U.S.TreasuryNoteCoupon2.88MaturityDate202808Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_JPMorganSecuritiesLLCCollateralizedbyParValue2800U.S.TreasuryNoteCoupon2.88MaturityDate202808Member" xlink:to="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue2800U.S.TreasuryNoteCoupon2.88MaturityDate202808Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Capitalgain_279020d8-89b9-4021-8e0a-26bb94581975_terseLabel_en-US" xlink:label="lab_efc_Capitalgain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Capital gain</link:label>
    <link:label id="lab_efc_Capitalgain_label_en-US" xlink:label="lab_efc_Capitalgain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Capital gain</link:label>
    <link:label id="lab_efc_Capitalgain_documentation_en-US" xlink:label="lab_efc_Capitalgain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The percentage of dividends which are classified as capital gains for tax purposes.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Capitalgain" xlink:href="efc-20201231.xsd#efc_Capitalgain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Capitalgain" xlink:to="lab_efc_Capitalgain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ElizonDB20151LLCMember_ebacf2b5-2da3-4fce-97d5-1a6705e06ed9_terseLabel_en-US" xlink:label="lab_efc_ElizonDB20151LLCMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Elizon DB 2015-1 LLC [Member]</link:label>
    <link:label id="lab_efc_ElizonDB20151LLCMember_label_en-US" xlink:label="lab_efc_ElizonDB20151LLCMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Elizon DB 2015-1 LLC [Member]</link:label>
    <link:label id="lab_efc_ElizonDB20151LLCMember_documentation_en-US" xlink:label="lab_efc_ElizonDB20151LLCMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Elizon DB 2015-1 LLC [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ElizonDB20151LLCMember" xlink:href="efc-20201231.xsd#efc_ElizonDB20151LLCMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ElizonDB20151LLCMember" xlink:to="lab_efc_ElizonDB20151LLCMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritizedloansMember_b50e2416-fdb8-4a64-8386-0f454ba62289_terseLabel_en-US" xlink:label="lab_efc_SecuritizedloansMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitized loans [Member]</link:label>
    <link:label id="lab_efc_SecuritizedloansMember_label_en-US" xlink:label="lab_efc_SecuritizedloansMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitized loans [Member]</link:label>
    <link:label id="lab_efc_SecuritizedloansMember_documentation_en-US" xlink:label="lab_efc_SecuritizedloansMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans securitized by the Company that have been consolidated by the Company.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritizedloansMember" xlink:href="efc-20201231.xsd#efc_SecuritizedloansMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritizedloansMember" xlink:to="lab_efc_SecuritizedloansMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PreferredStockParOrStatedValuePerShare_ecf8a89c-80c5-4567-a454-4f34b934b408_terseLabel_en-US" xlink:label="lab_us-gaap_PreferredStockParOrStatedValuePerShare" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Par or Stated Value Per Share</link:label>
    <link:label id="lab_us-gaap_PreferredStockParOrStatedValuePerShare_label_en-US" xlink:label="lab_us-gaap_PreferredStockParOrStatedValuePerShare" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Par or Stated Value Per Share</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PreferredStockParOrStatedValuePerShare" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PreferredStockParOrStatedValuePerShare"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PreferredStockParOrStatedValuePerShare" xlink:to="lab_us-gaap_PreferredStockParOrStatedValuePerShare" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FairValueOfBorrowingsNegativeYieldMeasurementInput_7992867e-2fec-4074-88a4-2d6088a188d7_negatedTerseLabel_en-US" xlink:label="lab_efc_FairValueOfBorrowingsNegativeYieldMeasurementInput" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value of borrowings, negative yield measurement input</link:label>
    <link:label id="lab_efc_FairValueOfBorrowingsNegativeYieldMeasurementInput_label_en-US" xlink:label="lab_efc_FairValueOfBorrowingsNegativeYieldMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value of borrowings, negative yield measurement input</link:label>
    <link:label id="lab_efc_FairValueOfBorrowingsNegativeYieldMeasurementInput_documentation_en-US" xlink:label="lab_efc_FairValueOfBorrowingsNegativeYieldMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value of borrowings, negative yield measurement input</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FairValueOfBorrowingsNegativeYieldMeasurementInput" xlink:href="efc-20201231.xsd#efc_FairValueOfBorrowingsNegativeYieldMeasurementInput"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FairValueOfBorrowingsNegativeYieldMeasurementInput" xlink:to="lab_efc_FairValueOfBorrowingsNegativeYieldMeasurementInput" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_GuaranteeObligationsCurrentCarryingValue_aa25e3c4-a70e-4c4e-9066-db772e16ff55_terseLabel_en-US" xlink:label="lab_us-gaap_GuaranteeObligationsCurrentCarryingValue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Guarantor Obligations, Current Carrying Value</link:label>
    <link:label id="lab_us-gaap_GuaranteeObligationsCurrentCarryingValue_label_en-US" xlink:label="lab_us-gaap_GuaranteeObligationsCurrentCarryingValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Guarantor Obligations, Current Carrying Value</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_GuaranteeObligationsCurrentCarryingValue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_GuaranteeObligationsCurrentCarryingValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_GuaranteeObligationsCurrentCarryingValue" xlink:to="lab_us-gaap_GuaranteeObligationsCurrentCarryingValue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RealEstateOwnedRollForward_d3d07273-c0be-40c5-b23e-59579d0fb8af_terseLabel_en-US" xlink:label="lab_efc_RealEstateOwnedRollForward" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Carrying Value</link:label>
    <link:label id="lab_efc_RealEstateOwnedRollForward_label_en-US" xlink:label="lab_efc_RealEstateOwnedRollForward" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Owned [Roll Forward]</link:label>
    <link:label id="lab_efc_RealEstateOwnedRollForward_documentation_en-US" xlink:label="lab_efc_RealEstateOwnedRollForward" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Owned [Roll Forward]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RealEstateOwnedRollForward" xlink:href="efc-20201231.xsd#efc_RealEstateOwnedRollForward"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RealEstateOwnedRollForward" xlink:to="lab_efc_RealEstateOwnedRollForward" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTable_feb9867b-b18e-4e2b-a483-0d0a0d2abbb6_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Accounts, Notes, Loans and Financing Receivable [Table]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTable_label_en-US" xlink:label="lab_us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Accounts, Notes, Loans and Financing Receivable [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTable" xlink:to="lab_us-gaap_ScheduleOfAccountsNotesLoansAndFinancingReceivableTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MaturityDateOfSecuritiesAsCollateralOnRepurchaseAgreement1_c19e71f1-ba34-4bf5-997b-95afad61d034_terseLabel_en-US" xlink:label="lab_efc_MaturityDateOfSecuritiesAsCollateralOnRepurchaseAgreement1" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity Date Of Securities As Collateral On Repurchase Agreement1</link:label>
    <link:label id="lab_efc_MaturityDateOfSecuritiesAsCollateralOnRepurchaseAgreement1_label_en-US" xlink:label="lab_efc_MaturityDateOfSecuritiesAsCollateralOnRepurchaseAgreement1" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity Date Of Securities As Collateral On Repurchase Agreement1</link:label>
    <link:label id="lab_efc_MaturityDateOfSecuritiesAsCollateralOnRepurchaseAgreement1_documentation_en-US" xlink:label="lab_efc_MaturityDateOfSecuritiesAsCollateralOnRepurchaseAgreement1" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity date of collateral underlying repurchase agreement.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MaturityDateOfSecuritiesAsCollateralOnRepurchaseAgreement1" xlink:href="efc-20201231.xsd#efc_MaturityDateOfSecuritiesAsCollateralOnRepurchaseAgreement1"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MaturityDateOfSecuritiesAsCollateralOnRepurchaseAgreement1" xlink:to="lab_efc_MaturityDateOfSecuritiesAsCollateralOnRepurchaseAgreement1" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_VestDecember132020Member_da45ae9e-8d11-4b2d-8401-592c2137371f_terseLabel_en-US" xlink:label="lab_efc_VestDecember132020Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 13, 2020 [Member]</link:label>
    <link:label id="lab_efc_VestDecember132020Member_label_en-US" xlink:label="lab_efc_VestDecember132020Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 13, 2020 [Member]</link:label>
    <link:label id="lab_efc_VestDecember132020Member_documentation_en-US" xlink:label="lab_efc_VestDecember132020Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 13, 2020 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_VestDecember132020Member" xlink:href="efc-20201231.xsd#efc_VestDecember132020Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_VestDecember132020Member" xlink:to="lab_efc_VestDecember132020Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationOtherliabilities_c4b593c4-9e24-4903-9033-73b9d06a9cb7_terseLabel_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationOtherliabilities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Other liabilities</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationOtherliabilities_label_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationOtherliabilities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Other liabilities</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationOtherliabilities_documentation_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationOtherliabilities" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The amount of other liabilities reported by an equity method investment of the entity.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationOtherliabilities" xlink:href="efc-20201231.xsd#efc_EquityMethodInvestmentSummarizedFinancialInformationOtherliabilities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationOtherliabilities" xlink:to="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationOtherliabilities" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilitiesAndEquity_c5528840-8106-4311-a422-050ff0d080bf_terseLabel_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilitiesAndEquity" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Liabilities and Equity</link:label>
    <link:label id="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilitiesAndEquity_label_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilitiesAndEquity" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Liabilities and Equity</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilitiesAndEquity" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilitiesAndEquity"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilitiesAndEquity" xlink:to="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilitiesAndEquity" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OperatingExpenses_4a3f1fd6-c80a-4c51-9b03-470688d049c9_totalLabel_en-US" xlink:label="lab_us-gaap_OperatingExpenses" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total expenses</link:label>
    <link:label id="lab_us-gaap_OperatingExpenses_fb169518-aa7f-4a3f-890b-22d742835b4a_terseLabel_en-US" xlink:label="lab_us-gaap_OperatingExpenses" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Expenses</link:label>
    <link:label id="lab_us-gaap_OperatingExpenses_label_en-US" xlink:label="lab_us-gaap_OperatingExpenses" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Expenses</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OperatingExpenses" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OperatingExpenses"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OperatingExpenses" xlink:to="lab_us-gaap_OperatingExpenses" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeCollateralObligationToReturnCash_95f2b4a3-b751-4213-b7aa-40414e079dc8_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_DerivativeCollateralObligationToReturnCash" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash Collateral (Received) Pledged</link:label>
    <link:label id="lab_us-gaap_DerivativeCollateralObligationToReturnCash_label_en-US" xlink:label="lab_us-gaap_DerivativeCollateralObligationToReturnCash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative, Collateral, Obligation to Return Cash</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeCollateralObligationToReturnCash" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeCollateralObligationToReturnCash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeCollateralObligationToReturnCash" xlink:to="lab_us-gaap_DerivativeCollateralObligationToReturnCash" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ToBeAnnouncedRMBSAbstract_e4a19a43-0980-46b8-997f-2270fa755208_terseLabel_en-US" xlink:label="lab_efc_ToBeAnnouncedRMBSAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">To Be Announced RMBS [Abstract]</link:label>
    <link:label id="lab_efc_ToBeAnnouncedRMBSAbstract_label_en-US" xlink:label="lab_efc_ToBeAnnouncedRMBSAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">To Be Announced RMBS [Abstract]</link:label>
    <link:label id="lab_efc_ToBeAnnouncedRMBSAbstract_documentation_en-US" xlink:label="lab_efc_ToBeAnnouncedRMBSAbstract" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">To Be Announced RMBS [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ToBeAnnouncedRMBSAbstract" xlink:href="efc-20201231.xsd#efc_ToBeAnnouncedRMBSAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ToBeAnnouncedRMBSAbstract" xlink:to="lab_efc_ToBeAnnouncedRMBSAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquityMethodInvestmentsTextBlock_a8615075-a47c-4997-847d-a2c8b0bbb88c_terseLabel_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentsTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Investments in Unconsolidated Entities</link:label>
    <link:label id="lab_us-gaap_EquityMethodInvestmentsTextBlock_label_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentsTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investments [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquityMethodInvestmentsTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquityMethodInvestmentsTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquityMethodInvestmentsTextBlock" xlink:to="lab_us-gaap_EquityMethodInvestmentsTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NonoperatingIncomeExpenseAbstract_da4535fa-2a2b-458d-917e-4083272dd92c_terseLabel_en-US" xlink:label="lab_us-gaap_NonoperatingIncomeExpenseAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Income (Loss)</link:label>
    <link:label id="lab_us-gaap_NonoperatingIncomeExpenseAbstract_label_en-US" xlink:label="lab_us-gaap_NonoperatingIncomeExpenseAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Nonoperating Income (Expense) [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NonoperatingIncomeExpenseAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NonoperatingIncomeExpenseAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NonoperatingIncomeExpenseAbstract" xlink:to="lab_us-gaap_NonoperatingIncomeExpenseAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CreditDefaultSwapCorporateBondsMember_746b1082-5610-49e3-ba16-d660939e95e5_terseLabel_en-US" xlink:label="lab_efc_CreditDefaultSwapCorporateBondsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit default swaps on corporate bonds</link:label>
    <link:label id="lab_efc_CreditDefaultSwapCorporateBondsMember_label_en-US" xlink:label="lab_efc_CreditDefaultSwapCorporateBondsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swap, Corporate Bonds [Member]</link:label>
    <link:label id="lab_efc_CreditDefaultSwapCorporateBondsMember_documentation_en-US" xlink:label="lab_efc_CreditDefaultSwapCorporateBondsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swap, Corporate Bonds [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CreditDefaultSwapCorporateBondsMember" xlink:href="efc-20201231.xsd#efc_CreditDefaultSwapCorporateBondsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CreditDefaultSwapCorporateBondsMember" xlink:to="lab_efc_CreditDefaultSwapCorporateBondsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LongInvestmentAratingMember_9ddcf074-8bf2-4ee6-819f-36f64e3e1e8a_terseLabel_en-US" xlink:label="lab_efc_LongInvestmentAratingMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Arating [Member]</link:label>
    <link:label id="lab_efc_LongInvestmentAratingMember_label_en-US" xlink:label="lab_efc_LongInvestmentAratingMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Arating [Member]</link:label>
    <link:label id="lab_efc_LongInvestmentAratingMember_documentation_en-US" xlink:label="lab_efc_LongInvestmentAratingMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">A rating as derived from Standard and Poor's, Moody's, and/or Fitch.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LongInvestmentAratingMember" xlink:href="efc-20201231.xsd#efc_LongInvestmentAratingMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LongInvestmentAratingMember" xlink:to="lab_efc_LongInvestmentAratingMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PurchaseofLoansNonCash_bcc78780-8cce-401c-8188-a6966e1ee9e3_terseLabel_en-US" xlink:label="lab_efc_PurchaseofLoansNonCash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchase of Loans Non Cash</link:label>
    <link:label id="lab_efc_PurchaseofLoansNonCash_label_en-US" xlink:label="lab_efc_PurchaseofLoansNonCash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchase of Loans Non Cash</link:label>
    <link:label id="lab_efc_PurchaseofLoansNonCash_documentation_en-US" xlink:label="lab_efc_PurchaseofLoansNonCash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchase of Loans Non Cash</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PurchaseofLoansNonCash" xlink:href="efc-20201231.xsd#efc_PurchaseofLoansNonCash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PurchaseofLoansNonCash" xlink:to="lab_efc_PurchaseofLoansNonCash" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesSoldNotYetPurchasedMember_05682476-b33e-411f-a083-3a3a1b2c21e3_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesSoldNotYetPurchasedMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments Sold Short</link:label>
    <link:label id="lab_us-gaap_SecuritiesSoldNotYetPurchasedMember_label_en-US" xlink:label="lab_us-gaap_SecuritiesSoldNotYetPurchasedMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Sold, Not yet Purchased [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesSoldNotYetPurchasedMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesSoldNotYetPurchasedMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesSoldNotYetPurchasedMember" xlink:to="lab_us-gaap_SecuritiesSoldNotYetPurchasedMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AgencySecuritiesMember_2085c14d-55ed-40ee-a602-7ff3046a72a8_terseLabel_en-US" xlink:label="lab_us-gaap_AgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Agency Securities [Member]</link:label>
    <link:label id="lab_us-gaap_AgencySecuritiesMember_label_en-US" xlink:label="lab_us-gaap_AgencySecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Agency Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AgencySecuritiesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AgencySecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AgencySecuritiesMember" xlink:to="lab_us-gaap_AgencySecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CreditDerivativesByContractTypeAxis_8766fdf0-6558-42a9-a5b3-dc23f873d839_terseLabel_en-US" xlink:label="lab_us-gaap_CreditDerivativesByContractTypeAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Derivatives Contract Type [Axis]</link:label>
    <link:label id="lab_us-gaap_CreditDerivativesByContractTypeAxis_label_en-US" xlink:label="lab_us-gaap_CreditDerivativesByContractTypeAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Derivatives Contract Type [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CreditDerivativesByContractTypeAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CreditDerivativesByContractTypeAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CreditDerivativesByContractTypeAxis" xlink:to="lab_us-gaap_CreditDerivativesByContractTypeAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MoneyMarketFundsatCost_e303130d-acf8-48f8-ba96-fb0341b266b4_terseLabel_en-US" xlink:label="lab_efc_MoneyMarketFundsatCost" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Money Market Funds, at Cost</link:label>
    <link:label id="lab_efc_MoneyMarketFundsatCost_label_en-US" xlink:label="lab_efc_MoneyMarketFundsatCost" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Money Market Funds, at Cost</link:label>
    <link:label id="lab_efc_MoneyMarketFundsatCost_documentation_en-US" xlink:label="lab_efc_MoneyMarketFundsatCost" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Represents cost basis of money market funds carried at fair value.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MoneyMarketFundsatCost" xlink:href="efc-20201231.xsd#efc_MoneyMarketFundsatCost"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MoneyMarketFundsatCost" xlink:to="lab_efc_MoneyMarketFundsatCost" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PutOptionMember_5dcf738d-0b3f-4995-b711-d525410c0fde_terseLabel_en-US" xlink:label="lab_us-gaap_PutOptionMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Put Option [Member]</link:label>
    <link:label id="lab_us-gaap_PutOptionMember_label_en-US" xlink:label="lab_us-gaap_PutOptionMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Put Option [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PutOptionMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PutOptionMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PutOptionMember" xlink:to="lab_us-gaap_PutOptionMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAsset_064bed3c-8c45-458a-bddb-483c7a4e478a_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAsset" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Sold under Agreements to Repurchase, Asset</link:label>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAsset_label_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAsset" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Sold under Agreements to Repurchase, Asset</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAsset" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAsset"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAsset" xlink:to="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseAsset" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EarningsPerShareBasicAndDiluted_6b7062d8-3b1d-4a22-9176-0cdd372554ca_terseLabel_en-US" xlink:label="lab_us-gaap_EarningsPerShareBasicAndDiluted" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Basic and Diluted (USD per share)</link:label>
    <link:label id="lab_us-gaap_EarningsPerShareBasicAndDiluted_label_en-US" xlink:label="lab_us-gaap_EarningsPerShareBasicAndDiluted" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share, Basic and Diluted</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EarningsPerShareBasicAndDiluted" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EarningsPerShareBasicAndDiluted"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EarningsPerShareBasicAndDiluted" xlink:to="lab_us-gaap_EarningsPerShareBasicAndDiluted" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StockIssuedDuringPeriodSharesIssuedForServices_5ead2adb-1129-4f77-bd5a-3adcb14f951d_terseLabel_en-US" xlink:label="lab_us-gaap_StockIssuedDuringPeriodSharesIssuedForServices" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Shares issued in connection with incentive fee payment</link:label>
    <link:label id="lab_us-gaap_StockIssuedDuringPeriodSharesIssuedForServices_label_en-US" xlink:label="lab_us-gaap_StockIssuedDuringPeriodSharesIssuedForServices" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stock Issued During Period, Shares, Issued for Services</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StockIssuedDuringPeriodSharesIssuedForServices" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockIssuedDuringPeriodSharesIssuedForServices"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StockIssuedDuringPeriodSharesIssuedForServices" xlink:to="lab_us-gaap_StockIssuedDuringPeriodSharesIssuedForServices" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationMember_7d132cd4-03e2-441e-a6b0-12c85da6e38d_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointSevenFivePercentMember_0418e2e8-9ed4-4fb6-8c82-5e443bc9868a_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointSevenFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Three Point Seven Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointSevenFivePercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointSevenFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Three Point Seven Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointSevenFivePercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointSevenFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Three Point Seven Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointSevenFivePercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointSevenFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointSevenFivePercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointSevenFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtDisclosureAbstract_61f2da2d-b39b-4a81-b51c-f244490854cf_terseLabel_en-US" xlink:label="lab_us-gaap_DebtDisclosureAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Disclosure [Abstract]</link:label>
    <link:label id="lab_us-gaap_DebtDisclosureAbstract_label_en-US" xlink:label="lab_us-gaap_DebtDisclosureAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Disclosure [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtDisclosureAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtDisclosureAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtDisclosureAbstract" xlink:to="lab_us-gaap_DebtDisclosureAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfMaturitiesOfLongTermDebtTableTextBlock_9a36e15d-d053-4d01-b78b-7cac21ee71d0_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfMaturitiesOfLongTermDebtTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Maturities of Long-term Debt [Table Text Block]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfMaturitiesOfLongTermDebtTableTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfMaturitiesOfLongTermDebtTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Maturities of Long-term Debt [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfMaturitiesOfLongTermDebtTableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfMaturitiesOfLongTermDebtTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfMaturitiesOfLongTermDebtTableTextBlock" xlink:to="lab_us-gaap_ScheduleOfMaturitiesOfLongTermDebtTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PreferredStockLiquidationPreference_c621b3e5-adea-4fba-8409-8b7039bf33c4_terseLabel_en-US" xlink:label="lab_us-gaap_PreferredStockLiquidationPreference" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Liquidation Preference Per Share</link:label>
    <link:label id="lab_us-gaap_PreferredStockLiquidationPreference_label_en-US" xlink:label="lab_us-gaap_PreferredStockLiquidationPreference" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Preferred Stock, Liquidation Preference Per Share</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PreferredStockLiquidationPreference" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PreferredStockLiquidationPreference"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PreferredStockLiquidationPreference" xlink:to="lab_us-gaap_PreferredStockLiquidationPreference" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MortgageOriginatorMember_8ef757eb-7a7b-4b7f-96f0-490714c5e470_terseLabel_en-US" xlink:label="lab_efc_MortgageOriginatorMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage Originator</link:label>
    <link:label id="lab_efc_MortgageOriginatorMember_label_en-US" xlink:label="lab_efc_MortgageOriginatorMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage Originator [Member]</link:label>
    <link:label id="lab_efc_MortgageOriginatorMember_documentation_en-US" xlink:label="lab_efc_MortgageOriginatorMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Mortgage Originator [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MortgageOriginatorMember" xlink:href="efc-20201231.xsd#efc_MortgageOriginatorMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MortgageOriginatorMember" xlink:to="lab_efc_MortgageOriginatorMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Minimumeconomicinterestrequiredtoberetained_34a2bfc5-036c-46ab-bfac-fa804feb8244_terseLabel_en-US" xlink:label="lab_efc_Minimumeconomicinterestrequiredtoberetained" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Minimum economic interest required to be retained</link:label>
    <link:label id="lab_efc_Minimumeconomicinterestrequiredtoberetained_label_en-US" xlink:label="lab_efc_Minimumeconomicinterestrequiredtoberetained" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Minimum economic interest required to be retained</link:label>
    <link:label id="lab_efc_Minimumeconomicinterestrequiredtoberetained_documentation_en-US" xlink:label="lab_efc_Minimumeconomicinterestrequiredtoberetained" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Minimum amount of economic interest required to be retained in securitization under risk retention rules.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Minimumeconomicinterestrequiredtoberetained" xlink:href="efc-20201231.xsd#efc_Minimumeconomicinterestrequiredtoberetained"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Minimumeconomicinterestrequiredtoberetained" xlink:to="lab_efc_Minimumeconomicinterestrequiredtoberetained" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MarketApproachValuationTechniqueMember_4cf43dff-9d42-40d4-bf38-0df83a63de11_terseLabel_en-US" xlink:label="lab_us-gaap_MarketApproachValuationTechniqueMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Valuation, Market Approach</link:label>
    <link:label id="lab_us-gaap_MarketApproachValuationTechniqueMember_label_en-US" xlink:label="lab_us-gaap_MarketApproachValuationTechniqueMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Valuation, Market Approach [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MarketApproachValuationTechniqueMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MarketApproachValuationTechniqueMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MarketApproachValuationTechniqueMember" xlink:to="lab_us-gaap_MarketApproachValuationTechniqueMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RealizedInvestmentGainsLosses_42eaf2d4-8230-46bd-8feb-bb2a5e26f8e3_terseLabel_en-US" xlink:label="lab_us-gaap_RealizedInvestmentGainsLosses" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments</link:label>
    <link:label id="lab_us-gaap_RealizedInvestmentGainsLosses_label_en-US" xlink:label="lab_us-gaap_RealizedInvestmentGainsLosses" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized Investment Gains (Losses)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RealizedInvestmentGainsLosses" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RealizedInvestmentGainsLosses"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RealizedInvestmentGainsLosses" xlink:to="lab_us-gaap_RealizedInvestmentGainsLosses" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_VariableInterestEntityLineItems_c20f37fc-570e-4a41-8f2b-92164655d61d_terseLabel_en-US" xlink:label="lab_us-gaap_VariableInterestEntityLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Variable Interest Entity [Line Items]</link:label>
    <link:label id="lab_us-gaap_VariableInterestEntityLineItems_label_en-US" xlink:label="lab_us-gaap_VariableInterestEntityLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Variable Interest Entity [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_VariableInterestEntityLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_VariableInterestEntityLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_VariableInterestEntityLineItems" xlink:to="lab_us-gaap_VariableInterestEntityLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationFivePointFIvePercentMember_b0d1ab3c-6b6e-4de8-b8d1-e6c7904ad7a9_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationFivePointFIvePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Five Point FIve Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationFivePointFIvePercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationFivePointFIvePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Five Point FIve Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationFivePointFIvePercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationFivePointFIvePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Five Point FIve Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationFivePointFIvePercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationFivePointFIvePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationFivePointFIvePercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationFivePointFIvePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PerformingResidentialMortgageLoansMember_69fce0d1-5ec3-46d0-b803-da987f9aa17c_terseLabel_en-US" xlink:label="lab_efc_PerformingResidentialMortgageLoansMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Performing residential mortgage loans</link:label>
    <link:label id="lab_efc_PerformingResidentialMortgageLoansMember_label_en-US" xlink:label="lab_efc_PerformingResidentialMortgageLoansMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Performing Residential Mortgage Loans [Member]</link:label>
    <link:label id="lab_efc_PerformingResidentialMortgageLoansMember_documentation_en-US" xlink:label="lab_efc_PerformingResidentialMortgageLoansMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Performing Residential Mortgage Loans [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PerformingResidentialMortgageLoansMember" xlink:href="efc-20201231.xsd#efc_PerformingResidentialMortgageLoansMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PerformingResidentialMortgageLoansMember" xlink:to="lab_efc_PerformingResidentialMortgageLoansMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DeferredFederalIncomeTaxExpenseBenefit_c45b621d-426a-4576-89eb-bf8cf529a22f_terseLabel_en-US" xlink:label="lab_us-gaap_DeferredFederalIncomeTaxExpenseBenefit" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Federal Income Tax Expense (Benefit)</link:label>
    <link:label id="lab_us-gaap_DeferredFederalIncomeTaxExpenseBenefit_label_en-US" xlink:label="lab_us-gaap_DeferredFederalIncomeTaxExpenseBenefit" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Federal Income Tax Expense (Benefit)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DeferredFederalIncomeTaxExpenseBenefit" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DeferredFederalIncomeTaxExpenseBenefit"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DeferredFederalIncomeTaxExpenseBenefit" xlink:to="lab_us-gaap_DeferredFederalIncomeTaxExpenseBenefit" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PastDueFinancingReceivablesTableTextBlock_044f886f-6bfb-41e9-ba6e-4e37201e1a53_terseLabel_en-US" xlink:label="lab_us-gaap_PastDueFinancingReceivablesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable, Past Due [Table Text Block]</link:label>
    <link:label id="lab_us-gaap_PastDueFinancingReceivablesTableTextBlock_label_en-US" xlink:label="lab_us-gaap_PastDueFinancingReceivablesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable, Past Due [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PastDueFinancingReceivablesTableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PastDueFinancingReceivablesTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PastDueFinancingReceivablesTableTextBlock" xlink:to="lab_us-gaap_PastDueFinancingReceivablesTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ReverseRepurchaseAgreementsMaturity_bd5a5847-4686-4835-b674-fe68302d4287_terseLabel_en-US" xlink:label="lab_efc_ReverseRepurchaseAgreementsMaturity" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reverse Repurchase Agreements Maturity</link:label>
    <link:label id="lab_efc_ReverseRepurchaseAgreementsMaturity_label_en-US" xlink:label="lab_efc_ReverseRepurchaseAgreementsMaturity" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reverse Repurchase Agreements Maturity</link:label>
    <link:label id="lab_efc_ReverseRepurchaseAgreementsMaturity_documentation_en-US" xlink:label="lab_efc_ReverseRepurchaseAgreementsMaturity" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Reverse Repurchase Agreements Maturity</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ReverseRepurchaseAgreementsMaturity" xlink:href="efc-20201231.xsd#efc_ReverseRepurchaseAgreementsMaturity"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ReverseRepurchaseAgreementsMaturity" xlink:to="lab_efc_ReverseRepurchaseAgreementsMaturity" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritizationTransactionsTableTextBlock_8bfa24d8-63a7-4ac2-bf59-bcb88f7ddee2_terseLabel_en-US" xlink:label="lab_efc_SecuritizationTransactionsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization Transactions</link:label>
    <link:label id="lab_efc_SecuritizationTransactionsTableTextBlock_label_en-US" xlink:label="lab_efc_SecuritizationTransactionsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization Transactions [Table Text Block]</link:label>
    <link:label id="lab_efc_SecuritizationTransactionsTableTextBlock_documentation_en-US" xlink:label="lab_efc_SecuritizationTransactionsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Table Text Block] for Securitization Transactions [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritizationTransactionsTableTextBlock" xlink:href="efc-20201231.xsd#efc_SecuritizationTransactionsTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritizationTransactionsTableTextBlock" xlink:to="lab_efc_SecuritizationTransactionsTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StatementOfFinancialPositionAbstract_80a1d499-3e5c-4fd0-ad16-f79ec034a19e_terseLabel_en-US" xlink:label="lab_us-gaap_StatementOfFinancialPositionAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Statement of Financial Position [Abstract]</link:label>
    <link:label id="lab_us-gaap_StatementOfFinancialPositionAbstract_label_en-US" xlink:label="lab_us-gaap_StatementOfFinancialPositionAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Statement of Financial Position [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StatementOfFinancialPositionAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StatementOfFinancialPositionAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StatementOfFinancialPositionAbstract" xlink:to="lab_us-gaap_StatementOfFinancialPositionAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DividedsPaidPerCommonShare_ec2ad0df-863e-4881-88dc-a20c83f93e6b_negatedTerseLabel_en-US" xlink:label="lab_efc_DividedsPaidPerCommonShare" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Divideds Paid Per Common Share</link:label>
    <link:label id="lab_efc_DividedsPaidPerCommonShare_label_en-US" xlink:label="lab_efc_DividedsPaidPerCommonShare" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Divideds Paid Per Common Share</link:label>
    <link:label id="lab_efc_DividedsPaidPerCommonShare_documentation_en-US" xlink:label="lab_efc_DividedsPaidPerCommonShare" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Per share dividend amount paid to common shareholders.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DividedsPaidPerCommonShare" xlink:href="efc-20201231.xsd#efc_DividedsPaidPerCommonShare"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DividedsPaidPerCommonShare" xlink:to="lab_efc_DividedsPaidPerCommonShare" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtInstrumentRedemptionPricePercentage_b9a1ed55-7292-4e55-992e-f704051ad147_terseLabel_en-US" xlink:label="lab_us-gaap_DebtInstrumentRedemptionPricePercentage" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Redemption percentage</link:label>
    <link:label id="lab_us-gaap_DebtInstrumentRedemptionPricePercentage_label_en-US" xlink:label="lab_us-gaap_DebtInstrumentRedemptionPricePercentage" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Instrument, Redemption Price, Percentage</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtInstrumentRedemptionPricePercentage" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtInstrumentRedemptionPricePercentage"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtInstrumentRedemptionPricePercentage" xlink:to="lab_us-gaap_DebtInstrumentRedemptionPricePercentage" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LoanaccrualstatusDomain_042c8226-ec41-4ada-869b-49c5ad98d2ee_terseLabel_en-US" xlink:label="lab_efc_LoanaccrualstatusDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loan accrual status [Domain]</link:label>
    <link:label id="lab_efc_LoanaccrualstatusDomain_label_en-US" xlink:label="lab_efc_LoanaccrualstatusDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loan accrual status [Domain]</link:label>
    <link:label id="lab_efc_LoanaccrualstatusDomain_documentation_en-US" xlink:label="lab_efc_LoanaccrualstatusDomain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Domain] for Loan accrual status [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LoanaccrualstatusDomain" xlink:href="efc-20201231.xsd#efc_LoanaccrualstatusDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LoanaccrualstatusDomain" xlink:to="lab_efc_LoanaccrualstatusDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OperatingPartnershipUnits_b495d0b5-16ef-4450-943a-7181a7c78628_terseLabel_en-US" xlink:label="lab_efc_OperatingPartnershipUnits" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership Units (in units)</link:label>
    <link:label id="lab_efc_OperatingPartnershipUnits_label_en-US" xlink:label="lab_efc_OperatingPartnershipUnits" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership Units</link:label>
    <link:label id="lab_efc_OperatingPartnershipUnits_documentation_en-US" xlink:label="lab_efc_OperatingPartnershipUnits" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership Units</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OperatingPartnershipUnits" xlink:href="efc-20201231.xsd#efc_OperatingPartnershipUnits"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OperatingPartnershipUnits" xlink:to="lab_efc_OperatingPartnershipUnits" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleOfRealEstateOwnedTableTextBlock_5de32819-a329-4c9a-823a-7cdb219bcb5f_terseLabel_en-US" xlink:label="lab_efc_ScheduleOfRealEstateOwnedTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Real Estate Owned</link:label>
    <link:label id="lab_efc_ScheduleOfRealEstateOwnedTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleOfRealEstateOwnedTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Real Estate Owned [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleOfRealEstateOwnedTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleOfRealEstateOwnedTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Real Estate Owned [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleOfRealEstateOwnedTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleOfRealEstateOwnedTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleOfRealEstateOwnedTableTextBlock" xlink:to="lab_efc_ScheduleOfRealEstateOwnedTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LIBOROASMember_723d3368-a2d1-4189-a20c-c9d8a25f4763_terseLabel_en-US" xlink:label="lab_efc_LIBOROASMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, LIBOR OAS</link:label>
    <link:label id="lab_efc_LIBOROASMember_label_en-US" xlink:label="lab_efc_LIBOROASMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">LIBOR OAS [Member]</link:label>
    <link:label id="lab_efc_LIBOROASMember_documentation_en-US" xlink:label="lab_efc_LIBOROASMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">LIBOR OAS [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LIBOROASMember" xlink:href="efc-20201231.xsd#efc_LIBOROASMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LIBOROASMember" xlink:to="lab_efc_LIBOROASMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue9161SovereignGovernmentBondCoupon2.75MaturityDate201904Member_10243e38-b6ae-4332-8546-8ea98ba9b873_terseLabel_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue9161SovereignGovernmentBondCoupon2.75MaturityDate201904Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $9,161 Sovereign Government Bond, Coupon 2.75%, Maturity Date 2019 04 [Member]</link:label>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue9161SovereignGovernmentBondCoupon2.75MaturityDate201904Member_label_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue9161SovereignGovernmentBondCoupon2.75MaturityDate201904Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $9,161 Sovereign Government Bond, Coupon 2.75%, Maturity Date 2019 04 [Member]</link:label>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue9161SovereignGovernmentBondCoupon2.75MaturityDate201904Member_documentation_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue9161SovereignGovernmentBondCoupon2.75MaturityDate201904Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $9,161 Sovereign Government Bond, Coupon 2.75%, Maturity Date 2019 04 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_JPMorganSecuritiesLLCCollateralizedbyParValue9161SovereignGovernmentBondCoupon2.75MaturityDate201904Member" xlink:href="efc-20201231.xsd#efc_JPMorganSecuritiesLLCCollateralizedbyParValue9161SovereignGovernmentBondCoupon2.75MaturityDate201904Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_JPMorganSecuritiesLLCCollateralizedbyParValue9161SovereignGovernmentBondCoupon2.75MaturityDate201904Member" xlink:to="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue9161SovereignGovernmentBondCoupon2.75MaturityDate201904Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MinorityInterestTable_bcb21e9c-b19c-411a-a42f-4670a2296cf7_terseLabel_en-US" xlink:label="lab_us-gaap_MinorityInterestTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noncontrolling Interest [Table]</link:label>
    <link:label id="lab_us-gaap_MinorityInterestTable_label_en-US" xlink:label="lab_us-gaap_MinorityInterestTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noncontrolling Interest [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MinorityInterestTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MinorityInterestTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MinorityInterestTable" xlink:to="lab_us-gaap_MinorityInterestTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OperatingPartnershipUnitsMember_7aa491b3-20aa-4d24-9c54-a43068637edc_terseLabel_en-US" xlink:label="lab_efc_OperatingPartnershipUnitsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership Units [Member]</link:label>
    <link:label id="lab_efc_OperatingPartnershipUnitsMember_label_en-US" xlink:label="lab_efc_OperatingPartnershipUnitsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Operating Partnership Units [Member]</link:label>
    <link:label id="lab_efc_OperatingPartnershipUnitsMember_documentation_en-US" xlink:label="lab_efc_OperatingPartnershipUnitsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common units of the Operating Partnership (as defined by the Company) not held by the Company.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OperatingPartnershipUnitsMember" xlink:href="efc-20201231.xsd#efc_OperatingPartnershipUnitsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OperatingPartnershipUnitsMember" xlink:to="lab_efc_OperatingPartnershipUnitsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquityMethodInvestmentOwnershipPercentage_08f06c86-e01f-47a6-bf69-6454fcea0f66_terseLabel_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentOwnershipPercentage" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Ownership Percentage</link:label>
    <link:label id="lab_us-gaap_EquityMethodInvestmentOwnershipPercentage_label_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentOwnershipPercentage" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Ownership Percentage</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquityMethodInvestmentOwnershipPercentage" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquityMethodInvestmentOwnershipPercentage"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquityMethodInvestmentOwnershipPercentage" xlink:to="lab_us-gaap_EquityMethodInvestmentOwnershipPercentage" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Netincomelossattributabletoparentpershare_9bf0a2e5-703d-49ff-94b7-89f35dc820b0_terseLabel_en-US" xlink:label="lab_efc_Netincomelossattributabletoparentpershare" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Results of Operations Attributable to Shareholders' Equity</link:label>
    <link:label id="lab_efc_Netincomelossattributabletoparentpershare_label_en-US" xlink:label="lab_efc_Netincomelossattributabletoparentpershare" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">NetIncomeLossAttributableToParentPerShare</link:label>
    <link:label id="lab_efc_Netincomelossattributabletoparentpershare_documentation_en-US" xlink:label="lab_efc_Netincomelossattributabletoparentpershare" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The portion of profit or loss for the period, net of income taxes, which is attributable to the parent based on average common shares outstanding during the period.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Netincomelossattributabletoparentpershare" xlink:href="efc-20201231.xsd#efc_Netincomelossattributabletoparentpershare"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Netincomelossattributabletoparentpershare" xlink:to="lab_efc_Netincomelossattributabletoparentpershare" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseInterestRate_5c862081-9b4a-4ed4-925d-6462c563e700_terseLabel_en-US" xlink:label="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseInterestRate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase Agreements Interest Rate</link:label>
    <link:label id="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseInterestRate_label_en-US" xlink:label="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseInterestRate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets Sold under Agreements to Repurchase, Interest Rate</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AssetsSoldUnderAgreementsToRepurchaseInterestRate" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AssetsSoldUnderAgreementsToRepurchaseInterestRate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AssetsSoldUnderAgreementsToRepurchaseInterestRate" xlink:to="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseInterestRate" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3000000And3249999Member_f97ba2a1-4477-4835-b2c2-11e71ca761e3_terseLabel_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3000000And3249999Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $3,000,000 and $3,249,999</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3000000And3249999Member_label_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3000000And3249999Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $3,000,000 and $3,249,999 [Member]</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3000000And3249999Member_documentation_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3000000And3249999Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $3,000,000 and $3,249,999</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3000000And3249999Member" xlink:href="efc-20201231.xsd#efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3000000And3249999Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3000000And3249999Member" xlink:to="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween3000000And3249999Member" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_ScheduleOfEquityMethodInvestmentEquityMethodInvesteeNameAxis_375a883c-66bf-4bf3-808b-388219818483_terseLabel_en-US" xlink:label="lab_srt_ScheduleOfEquityMethodInvestmentEquityMethodInvesteeNameAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment, Name [Axis]</link:label>
    <link:label id="lab_srt_ScheduleOfEquityMethodInvestmentEquityMethodInvesteeNameAxis_label_en-US" xlink:label="lab_srt_ScheduleOfEquityMethodInvestmentEquityMethodInvesteeNameAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment, Name [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_ScheduleOfEquityMethodInvestmentEquityMethodInvesteeNameAxis" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_ScheduleOfEquityMethodInvestmentEquityMethodInvesteeNameAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_ScheduleOfEquityMethodInvestmentEquityMethodInvesteeNameAxis" xlink:to="lab_srt_ScheduleOfEquityMethodInvestmentEquityMethodInvesteeNameAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquityMethodInvestmentsFairValueDisclosure_ec6e7a6e-0a90-44b1-a66b-ea2126cdd03c_terseLabel_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentsFairValueDisclosure" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment in unconsolidated entities, at fair value</link:label>
    <link:label id="lab_us-gaap_EquityMethodInvestmentsFairValueDisclosure_label_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentsFairValueDisclosure" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investments, Fair Value Disclosure</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquityMethodInvestmentsFairValueDisclosure" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquityMethodInvestmentsFairValueDisclosure"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquityMethodInvestmentsFairValueDisclosure" xlink:to="lab_us-gaap_EquityMethodInvestmentsFairValueDisclosure" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansOnRealEstateLineItems_1415858e-341a-473c-8ef4-e88bf7862982_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate [Line Items]</link:label>
    <link:label id="lab_srt_MortgageLoansOnRealEstateLineItems_label_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansOnRealEstateLineItems" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansOnRealEstateLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansOnRealEstateLineItems" xlink:to="lab_srt_MortgageLoansOnRealEstateLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_GA_35160cc1-0059-4c99-a301-e69f6c7ccd25_terseLabel_en-US" xlink:label="lab_stpr_GA" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">GEORGIA</link:label>
    <link:label id="lab_stpr_GA_label_en-US" xlink:label="lab_stpr_GA" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">GEORGIA</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_GA" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_GA"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_GA" xlink:to="lab_stpr_GA" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StatementOfCashFlowsAbstract_3b52cb71-c13a-4951-8b34-4d485c072e53_terseLabel_en-US" xlink:label="lab_us-gaap_StatementOfCashFlowsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Statement of Cash Flows [Abstract]</link:label>
    <link:label id="lab_us-gaap_StatementOfCashFlowsAbstract_label_en-US" xlink:label="lab_us-gaap_StatementOfCashFlowsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Statement of Cash Flows [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StatementOfCashFlowsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StatementOfCashFlowsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StatementOfCashFlowsAbstract" xlink:to="lab_us-gaap_StatementOfCashFlowsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RestrictedCash_c6244fb8-9bd5-4910-8b64-8502caac4760_terseLabel_en-US" xlink:label="lab_us-gaap_RestrictedCash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Restricted cash</link:label>
    <link:label id="lab_us-gaap_RestrictedCash_label_en-US" xlink:label="lab_us-gaap_RestrictedCash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Restricted Cash</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RestrictedCash" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RestrictedCash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RestrictedCash" xlink:to="lab_us-gaap_RestrictedCash" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Faceamountofinvestmentspurchasedfromrelatedparty_bfb588a3-ca6a-4e8c-b560-5f58a1ee3f38_terseLabel_en-US" xlink:label="lab_efc_Faceamountofinvestmentspurchasedfromrelatedparty" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Face amount of investments purchased from related party</link:label>
    <link:label id="lab_efc_Faceamountofinvestmentspurchasedfromrelatedparty_label_en-US" xlink:label="lab_efc_Faceamountofinvestmentspurchasedfromrelatedparty" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Face amount of investments purchased from related party</link:label>
    <link:label id="lab_efc_Faceamountofinvestmentspurchasedfromrelatedparty_documentation_en-US" xlink:label="lab_efc_Faceamountofinvestmentspurchasedfromrelatedparty" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Face amount of investments purchased from related party.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Faceamountofinvestmentspurchasedfromrelatedparty" xlink:href="efc-20201231.xsd#efc_Faceamountofinvestmentspurchasedfromrelatedparty"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Faceamountofinvestmentspurchasedfromrelatedparty" xlink:to="lab_efc_Faceamountofinvestmentspurchasedfromrelatedparty" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquitySecuritiesIndustryMember_2236ba21-9ba2-41ef-9b97-36788a6325ef_terseLabel_en-US" xlink:label="lab_us-gaap_EquitySecuritiesIndustryMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Industry sector [Domain]</link:label>
    <link:label id="lab_us-gaap_EquitySecuritiesIndustryMember_label_en-US" xlink:label="lab_us-gaap_EquitySecuritiesIndustryMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Industry Sector [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquitySecuritiesIndustryMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquitySecuritiesIndustryMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquitySecuritiesIndustryMember" xlink:to="lab_us-gaap_EquitySecuritiesIndustryMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NoncontrollingInterestMember_2f8c35dd-a277-493f-9c86-57fd56a68c5d_terseLabel_en-US" xlink:label="lab_us-gaap_NoncontrollingInterestMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling Interest</link:label>
    <link:label id="lab_us-gaap_NoncontrollingInterestMember_label_en-US" xlink:label="lab_us-gaap_NoncontrollingInterestMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noncontrolling Interest [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NoncontrollingInterestMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NoncontrollingInterestMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NoncontrollingInterestMember" xlink:to="lab_us-gaap_NoncontrollingInterestMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2000000And2249999Member_8a13411c-646b-41ba-a5d0-a2f87fa2e9c1_terseLabel_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2000000And2249999Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $2,000,000 and $2,249,999</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2000000And2249999Member_label_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2000000And2249999Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $2,000,000 and $2,249,999 [Member]</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2000000And2249999Member_documentation_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2000000And2249999Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $2,000,000 and $2,249,999</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2000000And2249999Member" xlink:href="efc-20201231.xsd#efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2000000And2249999Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2000000And2249999Member" xlink:to="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween2000000And2249999Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_AccruedIncentiveFee_1b650032-e969-44d9-a74c-e3357fb7cb8a_terseLabel_en-US" xlink:label="lab_efc_AccruedIncentiveFee" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Incentive fee payable to affiliate</link:label>
    <link:label id="lab_efc_AccruedIncentiveFee_label_en-US" xlink:label="lab_efc_AccruedIncentiveFee" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accrued Incentive Fee</link:label>
    <link:label id="lab_efc_AccruedIncentiveFee_documentation_en-US" xlink:label="lab_efc_AccruedIncentiveFee" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Represents the accrued expense incurred relating to performance based fees payable to the Manager.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_AccruedIncentiveFee" xlink:href="efc-20201231.xsd#efc_AccruedIncentiveFee"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_AccruedIncentiveFee" xlink:to="lab_efc_AccruedIncentiveFee" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancialInstrumentsFinancialLiabilitiesBalanceSheetGroupingsAbstract_e1946f72-e330-459d-b8a0-b161ca269e83_terseLabel_en-US" xlink:label="lab_us-gaap_FinancialInstrumentsFinancialLiabilitiesBalanceSheetGroupingsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities:</link:label>
    <link:label id="lab_us-gaap_FinancialInstrumentsFinancialLiabilitiesBalanceSheetGroupingsAbstract_label_en-US" xlink:label="lab_us-gaap_FinancialInstrumentsFinancialLiabilitiesBalanceSheetGroupingsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instruments, Financial Liabilities, Balance Sheet Groupings [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancialInstrumentsFinancialLiabilitiesBalanceSheetGroupingsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancialInstrumentsFinancialLiabilitiesBalanceSheetGroupingsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancialInstrumentsFinancialLiabilitiesBalanceSheetGroupingsAbstract" xlink:to="lab_us-gaap_FinancialInstrumentsFinancialLiabilitiesBalanceSheetGroupingsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_TransfersAccountedForAsSecuredBorrowingsAssetsCarryingAmount_2ba0c62b-ae1d-47ae-86b7-292bdc9dd5f9_terseLabel_en-US" xlink:label="lab_us-gaap_TransfersAccountedForAsSecuredBorrowingsAssetsCarryingAmount" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value of Assets Transfered and Accounted for as Secured Borrowings</link:label>
    <link:label id="lab_us-gaap_TransfersAccountedForAsSecuredBorrowingsAssetsCarryingAmount_label_en-US" xlink:label="lab_us-gaap_TransfersAccountedForAsSecuredBorrowingsAssetsCarryingAmount" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transfers Accounted for as Secured Borrowings, Assets, Carrying Amount</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_TransfersAccountedForAsSecuredBorrowingsAssetsCarryingAmount" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_TransfersAccountedForAsSecuredBorrowingsAssetsCarryingAmount"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_TransfersAccountedForAsSecuredBorrowingsAssetsCarryingAmount" xlink:to="lab_us-gaap_TransfersAccountedForAsSecuredBorrowingsAssetsCarryingAmount" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_FL_0f2ce404-915a-4ac6-99f0-bd5ea9840152_terseLabel_en-US" xlink:label="lab_stpr_FL" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Florida [Member]</link:label>
    <link:label id="lab_stpr_FL_label_en-US" xlink:label="lab_stpr_FL" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">FLORIDA</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_FL" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_FL"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_FL" xlink:to="lab_stpr_FL" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SaleOfStockNameOfTransactionDomain_98fd4027-9486-4aad-b6d4-1d6aacc4a7b8_terseLabel_en-US" xlink:label="lab_us-gaap_SaleOfStockNameOfTransactionDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Sale of Stock [Domain]</link:label>
    <link:label id="lab_us-gaap_SaleOfStockNameOfTransactionDomain_label_en-US" xlink:label="lab_us-gaap_SaleOfStockNameOfTransactionDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Sale of Stock [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SaleOfStockNameOfTransactionDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SaleOfStockNameOfTransactionDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SaleOfStockNameOfTransactionDomain" xlink:to="lab_us-gaap_SaleOfStockNameOfTransactionDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ExpensesPolicyTextBlock_c10c946a-60bd-4e28-94a2-20b8e1bdea7a_terseLabel_en-US" xlink:label="lab_efc_ExpensesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Expenses</link:label>
    <link:label id="lab_efc_ExpensesPolicyTextBlock_label_en-US" xlink:label="lab_efc_ExpensesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Expenses [Policy Text Block]</link:label>
    <link:label id="lab_efc_ExpensesPolicyTextBlock_documentation_en-US" xlink:label="lab_efc_ExpensesPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The Company's policy for when expenses are recognized.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ExpensesPolicyTextBlock" xlink:href="efc-20201231.xsd#efc_ExpensesPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ExpensesPolicyTextBlock" xlink:to="lab_efc_ExpensesPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ProceedsreceivedfromOthersecuredborrowingsatfairvaluenoncash_e7e4ae5f-d4cb-45bc-a04a-da7c50f5f0d4_terseLabel_en-US" xlink:label="lab_efc_ProceedsreceivedfromOthersecuredborrowingsatfairvaluenoncash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds received from Other secured borrowings, at fair value (non-cash)</link:label>
    <link:label id="lab_efc_ProceedsreceivedfromOthersecuredborrowingsatfairvaluenoncash_label_en-US" xlink:label="lab_efc_ProceedsreceivedfromOthersecuredborrowingsatfairvaluenoncash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds received from Other secured borrowings, at fair value (non-cash)</link:label>
    <link:label id="lab_efc_ProceedsreceivedfromOthersecuredborrowingsatfairvaluenoncash_documentation_en-US" xlink:label="lab_efc_ProceedsreceivedfromOthersecuredborrowingsatfairvaluenoncash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds received from Other secured borrowings, at fair value (non-cash)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ProceedsreceivedfromOthersecuredborrowingsatfairvaluenoncash" xlink:href="efc-20201231.xsd#efc_ProceedsreceivedfromOthersecuredborrowingsatfairvaluenoncash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ProceedsreceivedfromOthersecuredborrowingsatfairvaluenoncash" xlink:to="lab_efc_ProceedsreceivedfromOthersecuredborrowingsatfairvaluenoncash" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfEarningsPerShareBasicAndDilutedTableTextBlock_d39dc9c7-bfef-4ca8-a1fb-85cab7e6db04_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfEarningsPerShareBasicAndDilutedTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Computation Of Basic And Diluted EPS</link:label>
    <link:label id="lab_us-gaap_ScheduleOfEarningsPerShareBasicAndDilutedTableTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfEarningsPerShareBasicAndDilutedTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Earnings Per Share, Basic and Diluted [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfEarningsPerShareBasicAndDilutedTableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfEarningsPerShareBasicAndDilutedTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfEarningsPerShareBasicAndDilutedTableTextBlock" xlink:to="lab_us-gaap_ScheduleOfEarningsPerShareBasicAndDilutedTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureTextBlock_cd600ea3-4451-44dd-9295-3761b92cd1cc_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Derivatives</link:label>
    <link:label id="lab_us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureTextBlock_label_en-US" xlink:label="lab_us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Instruments and Hedging Activities Disclosure [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureTextBlock" xlink:to="lab_us-gaap_DerivativeInstrumentsAndHedgingActivitiesDisclosureTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year5Point50PercentMember_81c46691-97ae-423f-8d26-2c05977bc0f6_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year5Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 5 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year5Point50PercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year5Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 5 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year5Point50PercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year5Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 5 Point 50 Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociation30Year5Point50PercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociation30Year5Point50PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociation30Year5Point50PercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociation30Year5Point50PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePointFivePercentMember_423480b6-0514-47cb-af4e-768289323944_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Five Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Five Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Five Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePointFivePercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FairValueImpairedLoans_c44233b7-1ca6-439c-8501-0093bf2104a1_terseLabel_en-US" xlink:label="lab_efc_FairValueImpairedLoans" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Impaired Loans</link:label>
    <link:label id="lab_efc_FairValueImpairedLoans_label_en-US" xlink:label="lab_efc_FairValueImpairedLoans" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Impaired Loans</link:label>
    <link:label id="lab_efc_FairValueImpairedLoans_documentation_en-US" xlink:label="lab_efc_FairValueImpairedLoans" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value of loans that incurred realized losses from impairment charges.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FairValueImpairedLoans" xlink:href="efc-20201231.xsd#efc_FairValueImpairedLoans"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FairValueImpairedLoans" xlink:to="lab_efc_FairValueImpairedLoans" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AvailableForSaleSecuritiesGrossRealizedGains_0c6b5102-37be-47ea-ae13-969ce75dc495_terseLabel_en-US" xlink:label="lab_us-gaap_AvailableForSaleSecuritiesGrossRealizedGains" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Available-for-sale Securities, Gross Realized Gains</link:label>
    <link:label id="lab_us-gaap_AvailableForSaleSecuritiesGrossRealizedGains_label_en-US" xlink:label="lab_us-gaap_AvailableForSaleSecuritiesGrossRealizedGains" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Available-for-sale Securities, Gross Realized Gains</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AvailableForSaleSecuritiesGrossRealizedGains" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AvailableForSaleSecuritiesGrossRealizedGains"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AvailableForSaleSecuritiesGrossRealizedGains" xlink:to="lab_us-gaap_AvailableForSaleSecuritiesGrossRealizedGains" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralAtFairValue_4c651807-3702-4404-b009-39354fe99fc2_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralAtFairValue" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instruments Transferred or Pledged as Collateral</link:label>
    <link:label id="lab_us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralAtFairValue_label_en-US" xlink:label="lab_us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralAtFairValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instruments, Owned and Pledged as Collateral, at Fair Value</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralAtFairValue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralAtFairValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralAtFairValue" xlink:to="lab_us-gaap_FinancialInstrumentsOwnedAndPledgedAsCollateralAtFairValue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ForwardstartinginterestrateswapMember_9657c9fe-c822-4463-9f79-5945dbf51afb_terseLabel_en-US" xlink:label="lab_efc_ForwardstartinginterestrateswapMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Forward-starting interest rate swap [Member]</link:label>
    <link:label id="lab_efc_ForwardstartinginterestrateswapMember_label_en-US" xlink:label="lab_efc_ForwardstartinginterestrateswapMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Forward-starting interest rate swap [Member]</link:label>
    <link:label id="lab_efc_ForwardstartinginterestrateswapMember_documentation_en-US" xlink:label="lab_efc_ForwardstartinginterestrateswapMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Forward-starting interest rate swap [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ForwardstartinginterestrateswapMember" xlink:href="efc-20201231.xsd#efc_ForwardstartinginterestrateswapMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ForwardstartinginterestrateswapMember" xlink:to="lab_efc_ForwardstartinginterestrateswapMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FairValueInputsYieldIncludingNegative_87f0d047-49f5-4ea6-834c-0526d32aac7d_terseLabel_en-US" xlink:label="lab_efc_FairValueInputsYieldIncludingNegative" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Inputs, Yield Including Negative</link:label>
    <link:label id="lab_efc_FairValueInputsYieldIncludingNegative_label_en-US" xlink:label="lab_efc_FairValueInputsYieldIncludingNegative" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Inputs, Yield Including Negative</link:label>
    <link:label id="lab_efc_FairValueInputsYieldIncludingNegative_documentation_en-US" xlink:label="lab_efc_FairValueInputsYieldIncludingNegative" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Inputs, Yield Including Negative</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FairValueInputsYieldIncludingNegative" xlink:href="efc-20201231.xsd#efc_FairValueInputsYieldIncludingNegative"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FairValueInputsYieldIncludingNegative" xlink:to="lab_efc_FairValueInputsYieldIncludingNegative" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10102SovereignGovernmentBondCoupon0.75MaturityDate202107Member_ab43ceb1-9eff-4410-a4b9-124f7ebcf290_terseLabel_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10102SovereignGovernmentBondCoupon0.75MaturityDate202107Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $10,102 Sovereign Government Bond, Coupon 0.75%, Maturity Date 2021 07 [Member]</link:label>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10102SovereignGovernmentBondCoupon0.75MaturityDate202107Member_label_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10102SovereignGovernmentBondCoupon0.75MaturityDate202107Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $10,102 Sovereign Government Bond, Coupon 0.75%, Maturity Date 2021 07 [Member]</link:label>
    <link:label id="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10102SovereignGovernmentBondCoupon0.75MaturityDate202107Member_documentation_en-US" xlink:label="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10102SovereignGovernmentBondCoupon0.75MaturityDate202107Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">JP Morgan Securities LLC Collateralized by Par Value $10,102 Sovereign Government Bond, Coupon 0.75%, Maturity Date 2021 07 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10102SovereignGovernmentBondCoupon0.75MaturityDate202107Member" xlink:href="efc-20201231.xsd#efc_JPMorganSecuritiesLLCCollateralizedbyParValue10102SovereignGovernmentBondCoupon0.75MaturityDate202107Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10102SovereignGovernmentBondCoupon0.75MaturityDate202107Member" xlink:to="lab_efc_JPMorganSecuritiesLLCCollateralizedbyParValue10102SovereignGovernmentBondCoupon0.75MaturityDate202107Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseCollateralRightToReclaimCash_b3d717fb-0701-4f61-9f70-d408e0d8da5c_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseCollateralRightToReclaimCash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash Collateral (Received) Pledged</link:label>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseCollateralRightToReclaimCash_label_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseCollateralRightToReclaimCash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Sold under Agreements to Repurchase, Collateral, Right to Reclaim Cash</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseCollateralRightToReclaimCash" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseCollateralRightToReclaimCash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseCollateralRightToReclaimCash" xlink:to="lab_us-gaap_SecuritiesSoldUnderAgreementsToRepurchaseCollateralRightToReclaimCash" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityValue_45a0b642-e89c-4306-b544-1ea4c7517f1f_negatedPeriodStartLabel_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityValue" xlink:role="http://www.xbrl.org/2009/role/negatedPeriodStartLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities, Begining Balance</link:label>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityValue_13ac80e9-6d14-48c1-9f58-194174541a86_negatedPeriodEndLabel_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityValue" xlink:role="http://www.xbrl.org/2009/role/negatedPeriodEndLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities, Ending Balance</link:label>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityValue_label_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement with Unobservable Inputs Reconciliation, Recurring Basis, Liability Value</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityValue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityValue" xlink:to="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationsRecurringBasisLiabilityValue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_UnrealizedGainLossonFinancialDerivativesexcludingcurrencyhedges_3053c284-5e62-49b6-a886-28020c5b4cf9_terseLabel_en-US" xlink:label="lab_efc_UnrealizedGainLossonFinancialDerivativesexcludingcurrencyhedges" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Derivatives, excluding currency hedges</link:label>
    <link:label id="lab_efc_UnrealizedGainLossonFinancialDerivativesexcludingcurrencyhedges_label_en-US" xlink:label="lab_efc_UnrealizedGainLossonFinancialDerivativesexcludingcurrencyhedges" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized Gain (Loss) on Financial Derivatives, excluding currency hedges</link:label>
    <link:label id="lab_efc_UnrealizedGainLossonFinancialDerivativesexcludingcurrencyhedges_documentation_en-US" xlink:label="lab_efc_UnrealizedGainLossonFinancialDerivativesexcludingcurrencyhedges" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized Gain (Loss) on Financial Derivatives, excluding currency hedges</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_UnrealizedGainLossonFinancialDerivativesexcludingcurrencyhedges" xlink:href="efc-20201231.xsd#efc_UnrealizedGainLossonFinancialDerivativesexcludingcurrencyhedges"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_UnrealizedGainLossonFinancialDerivativesexcludingcurrencyhedges" xlink:to="lab_efc_UnrealizedGainLossonFinancialDerivativesexcludingcurrencyhedges" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncomeTaxDisclosureTextBlock_e200fa6e-245d-48ac-ad7d-4c78a7c81831_terseLabel_en-US" xlink:label="lab_us-gaap_IncomeTaxDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Income Taxes</link:label>
    <link:label id="lab_us-gaap_IncomeTaxDisclosureTextBlock_label_en-US" xlink:label="lab_us-gaap_IncomeTaxDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Income Tax Disclosure [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncomeTaxDisclosureTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncomeTaxDisclosureTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncomeTaxDisclosureTextBlock" xlink:to="lab_us-gaap_IncomeTaxDisclosureTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Unfundedcommitment_8571419b-4e88-429c-9a56-34084b7d0fe6_terseLabel_en-US" xlink:label="lab_efc_Unfundedcommitment" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unfunded commitment</link:label>
    <link:label id="lab_efc_Unfundedcommitment_label_en-US" xlink:label="lab_efc_Unfundedcommitment" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unfunded commitment</link:label>
    <link:label id="lab_efc_Unfundedcommitment_documentation_en-US" xlink:label="lab_efc_Unfundedcommitment" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount of unfunded commitment at period end.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Unfundedcommitment" xlink:href="efc-20201231.xsd#efc_Unfundedcommitment"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Unfundedcommitment" xlink:to="lab_efc_Unfundedcommitment" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LongInvestmentBbRatingMember_209827ba-b88e-439b-b013-b5567cd40593_terseLabel_en-US" xlink:label="lab_efc_LongInvestmentBbRatingMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Bb Rating [Member]</link:label>
    <link:label id="lab_efc_LongInvestmentBbRatingMember_label_en-US" xlink:label="lab_efc_LongInvestmentBbRatingMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long Investment Bb Rating [Member]</link:label>
    <link:label id="lab_efc_LongInvestmentBbRatingMember_documentation_en-US" xlink:label="lab_efc_LongInvestmentBbRatingMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">BB rating as derived from Standard and Poor's, Moody's, and/or Fitch.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LongInvestmentBbRatingMember" xlink:href="efc-20201231.xsd#efc_LongInvestmentBbRatingMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LongInvestmentBbRatingMember" xlink:to="lab_efc_LongInvestmentBbRatingMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodShares_056407d2-c0d0-4abb-b8f9-3ae390a3fd44_negatedLabel_en-US" xlink:label="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodShares" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase of shares of common stock (in shares)</link:label>
    <link:label id="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodShares_dfea6a3c-ffe6-42a5-aad4-925eadb82752_terseLabel_en-US" xlink:label="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodShares" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Shares repurchased (in shares)</link:label>
    <link:label id="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodShares_027e4c75-948f-4fe0-98df-bd0f7729f32c_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodShares" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Shares repurchased</link:label>
    <link:label id="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodShares_label_en-US" xlink:label="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodShares" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stock Repurchased and Retired During Period, Shares</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StockRepurchasedAndRetiredDuringPeriodShares" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockRepurchasedAndRetiredDuringPeriodShares"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StockRepurchasedAndRetiredDuringPeriodShares" xlink:to="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodShares" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DividendsPayableDateOfRecordDayMonthAndYear_89235ed0-478e-4fe0-8483-ab290a4c6804_terseLabel_en-US" xlink:label="lab_us-gaap_DividendsPayableDateOfRecordDayMonthAndYear" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Record Date</link:label>
    <link:label id="lab_us-gaap_DividendsPayableDateOfRecordDayMonthAndYear_label_en-US" xlink:label="lab_us-gaap_DividendsPayableDateOfRecordDayMonthAndYear" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends Payable, Date of Record</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DividendsPayableDateOfRecordDayMonthAndYear" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DividendsPayableDateOfRecordDayMonthAndYear"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DividendsPayableDateOfRecordDayMonthAndYear" xlink:to="lab_us-gaap_DividendsPayableDateOfRecordDayMonthAndYear" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems_c50c3463-a19a-4752-aa70-78af09a18fd0_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Liabilities Measured on Recurring Basis, Unobservable Input Reconciliation [Line Items]</link:label>
    <link:label id="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems_label_en-US" xlink:label="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Liabilities Measured on Recurring Basis, Unobservable Input Reconciliation [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems" xlink:to="lab_us-gaap_FairValueLiabilitiesMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Increasedecreaseininterestandprincipalreceivable_0fe77bee-2f2a-4e7a-8934-f358109704e3_negatedTerseLabel_en-US" xlink:label="lab_efc_Increasedecreaseininterestandprincipalreceivable" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase decrease in interest and principal receivable</link:label>
    <link:label id="lab_efc_Increasedecreaseininterestandprincipalreceivable_label_en-US" xlink:label="lab_efc_Increasedecreaseininterestandprincipalreceivable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase decrease in interest and principal receivable</link:label>
    <link:label id="lab_efc_Increasedecreaseininterestandprincipalreceivable_documentation_en-US" xlink:label="lab_efc_Increasedecreaseininterestandprincipalreceivable" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase decrease in interest and principal receivable</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Increasedecreaseininterestandprincipalreceivable" xlink:href="efc-20201231.xsd#efc_Increasedecreaseininterestandprincipalreceivable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Increasedecreaseininterestandprincipalreceivable" xlink:to="lab_efc_Increasedecreaseininterestandprincipalreceivable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToResellNotOffsetPolicyElectionDeduction_56b7f7bb-a918-4041-9433-342743559902_negatedLabel_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToResellNotOffsetPolicyElectionDeduction" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instruments Available for Offset</link:label>
    <link:label id="lab_us-gaap_SecuritiesSoldUnderAgreementsToResellNotOffsetPolicyElectionDeduction_label_en-US" xlink:label="lab_us-gaap_SecuritiesSoldUnderAgreementsToResellNotOffsetPolicyElectionDeduction" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Sold under Agreements to Repurchase, Not Offset, Policy Election Deduction</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesSoldUnderAgreementsToResellNotOffsetPolicyElectionDeduction" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesSoldUnderAgreementsToResellNotOffsetPolicyElectionDeduction"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesSoldUnderAgreementsToResellNotOffsetPolicyElectionDeduction" xlink:to="lab_us-gaap_SecuritiesSoldUnderAgreementsToResellNotOffsetPolicyElectionDeduction" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NetRealizedGains_ef56c78a-e782-42f7-b4a4-413d14a1a92d_terseLabel_en-US" xlink:label="lab_efc_NetRealizedGains" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net realized gains</link:label>
    <link:label id="lab_efc_NetRealizedGains_label_en-US" xlink:label="lab_efc_NetRealizedGains" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net realized gains</link:label>
    <link:label id="lab_efc_NetRealizedGains_documentation_en-US" xlink:label="lab_efc_NetRealizedGains" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net realized gains</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NetRealizedGains" xlink:href="efc-20201231.xsd#efc_NetRealizedGains"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NetRealizedGains" xlink:to="lab_efc_NetRealizedGains" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GoldmanSachsFinancialSquareFundsGovernmentFundMember_0511a40b-d81a-42b8-830e-e293f7315e22_terseLabel_en-US" xlink:label="lab_efc_GoldmanSachsFinancialSquareFundsGovernmentFundMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Goldman Sachs Financial Square Funds - Government Fund [Member]</link:label>
    <link:label id="lab_efc_GoldmanSachsFinancialSquareFundsGovernmentFundMember_label_en-US" xlink:label="lab_efc_GoldmanSachsFinancialSquareFundsGovernmentFundMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Goldman Sachs Financial Square Funds - Government Fund [Member]</link:label>
    <link:label id="lab_efc_GoldmanSachsFinancialSquareFundsGovernmentFundMember_documentation_en-US" xlink:label="lab_efc_GoldmanSachsFinancialSquareFundsGovernmentFundMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Goldman Sachs Financial Square Funds - Government Fund [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GoldmanSachsFinancialSquareFundsGovernmentFundMember" xlink:href="efc-20201231.xsd#efc_GoldmanSachsFinancialSquareFundsGovernmentFundMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GoldmanSachsFinancialSquareFundsGovernmentFundMember" xlink:to="lab_efc_GoldmanSachsFinancialSquareFundsGovernmentFundMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CashoutflowfordebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue_6006749a-77c9-4fb5-9141-bf7a23feea3c_negatedTerseLabel_en-US" xlink:label="lab_efc_CashoutflowfordebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt issuance costs related to Other secured borrowings, at fair value</link:label>
    <link:label id="lab_efc_CashoutflowfordebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue_label_en-US" xlink:label="lab_efc_CashoutflowfordebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash outflow for debt issuance costs related to Other secured borrowings at fair value</link:label>
    <link:label id="lab_efc_CashoutflowfordebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue_documentation_en-US" xlink:label="lab_efc_CashoutflowfordebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash outflow for debt issuance costs related to Other secured borrowings carried at fair value which were expensed fully at time of issuance.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CashoutflowfordebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue" xlink:href="efc-20201231.xsd#efc_CashoutflowfordebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CashoutflowfordebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue" xlink:to="lab_efc_CashoutflowfordebtissuancecostsrelatedtoOthersecuredborrowingsatfairvalue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Maturity91to120daysMember_a0efccb8-8275-44a8-b35a-63490ba68b28_terseLabel_en-US" xlink:label="lab_efc_Maturity91to120daysMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">91-120 Days</link:label>
    <link:label id="lab_efc_Maturity91to120daysMember_label_en-US" xlink:label="lab_efc_Maturity91to120daysMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity 91 to 120 days [Member]</link:label>
    <link:label id="lab_efc_Maturity91to120daysMember_documentation_en-US" xlink:label="lab_efc_Maturity91to120daysMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity 91 to 120 days [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Maturity91to120daysMember" xlink:href="efc-20201231.xsd#efc_Maturity91to120daysMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Maturity91to120daysMember" xlink:to="lab_efc_Maturity91to120daysMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_WarrantMember_b986a19b-8bc3-4a68-b844-a9ff0eea932c_terseLabel_en-US" xlink:label="lab_us-gaap_WarrantMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Warrants</link:label>
    <link:label id="lab_us-gaap_WarrantMember_label_en-US" xlink:label="lab_us-gaap_WarrantMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Warrant [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_WarrantMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_WarrantMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_WarrantMember" xlink:to="lab_us-gaap_WarrantMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_USGovernmentAgenciesDebtSecuritiesMember_04cee780-2312-46e3-a2a8-501fd13db3ad_terseLabel_en-US" xlink:label="lab_us-gaap_USGovernmentAgenciesDebtSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Agency RMBS</link:label>
    <link:label id="lab_us-gaap_USGovernmentAgenciesDebtSecuritiesMember_label_en-US" xlink:label="lab_us-gaap_USGovernmentAgenciesDebtSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">US Government Agencies Debt Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_USGovernmentAgenciesDebtSecuritiesMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_USGovernmentAgenciesDebtSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_USGovernmentAgenciesDebtSecuritiesMember" xlink:to="lab_us-gaap_USGovernmentAgenciesDebtSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OtherAdditionalCapital_6a9543a1-bffb-4f7e-8867-2c4e12d96617_terseLabel_en-US" xlink:label="lab_us-gaap_OtherAdditionalCapital" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Additional paid-in-capital</link:label>
    <link:label id="lab_us-gaap_OtherAdditionalCapital_label_en-US" xlink:label="lab_us-gaap_OtherAdditionalCapital" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Additional Capital</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OtherAdditionalCapital" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OtherAdditionalCapital"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OtherAdditionalCapital" xlink:to="lab_us-gaap_OtherAdditionalCapital" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentCompanyTotalReturnAfterIncentiveFees_40ef2be0-4b5a-449a-aecb-142c97ea56d1_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentCompanyTotalReturnAfterIncentiveFees" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Total Return after Incentive Fees</link:label>
    <link:label id="lab_us-gaap_InvestmentCompanyTotalReturnAfterIncentiveFees_label_en-US" xlink:label="lab_us-gaap_InvestmentCompanyTotalReturnAfterIncentiveFees" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Total Return after Incentive Fees</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentCompanyTotalReturnAfterIncentiveFees" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentCompanyTotalReturnAfterIncentiveFees"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentCompanyTotalReturnAfterIncentiveFees" xlink:to="lab_us-gaap_InvestmentCompanyTotalReturnAfterIncentiveFees" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ProceedsfromissuanceofOthersecuredborrowingsnoncash_b401d2f0-3a71-4c83-9d92-ae070fbbbc81_terseLabel_en-US" xlink:label="lab_efc_ProceedsfromissuanceofOthersecuredborrowingsnoncash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from issuance of Other secured borrowings (non-cash)</link:label>
    <link:label id="lab_efc_ProceedsfromissuanceofOthersecuredborrowingsnoncash_label_en-US" xlink:label="lab_efc_ProceedsfromissuanceofOthersecuredborrowingsnoncash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from issuance of Other secured borrowings (non-cash)</link:label>
    <link:label id="lab_efc_ProceedsfromissuanceofOthersecuredborrowingsnoncash_documentation_en-US" xlink:label="lab_efc_ProceedsfromissuanceofOthersecuredborrowingsnoncash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from issuance of Other secured borrowings (non-cash)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ProceedsfromissuanceofOthersecuredborrowingsnoncash" xlink:href="efc-20201231.xsd#efc_ProceedsfromissuanceofOthersecuredborrowingsnoncash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ProceedsfromissuanceofOthersecuredborrowingsnoncash" xlink:to="lab_efc_ProceedsfromissuanceofOthersecuredborrowingsnoncash" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DerivativeMaturityPeriodAxis_6b0256fc-180d-4400-b3ec-01a639cbc1e1_terseLabel_en-US" xlink:label="lab_efc_DerivativeMaturityPeriodAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Maturity Period [Axis]</link:label>
    <link:label id="lab_efc_DerivativeMaturityPeriodAxis_label_en-US" xlink:label="lab_efc_DerivativeMaturityPeriodAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Maturity Period [Axis]</link:label>
    <link:label id="lab_efc_DerivativeMaturityPeriodAxis_documentation_en-US" xlink:label="lab_efc_DerivativeMaturityPeriodAxis" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Maturity Period [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DerivativeMaturityPeriodAxis" xlink:href="efc-20201231.xsd#efc_DerivativeMaturityPeriodAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DerivativeMaturityPeriodAxis" xlink:to="lab_efc_DerivativeMaturityPeriodAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ManagementFeeExpenseGross_3b2335db-a6ee-499b-ae38-deb486961d83_terseLabel_en-US" xlink:label="lab_efc_ManagementFeeExpenseGross" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Base management fee, gross</link:label>
    <link:label id="lab_efc_ManagementFeeExpenseGross_label_en-US" xlink:label="lab_efc_ManagementFeeExpenseGross" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Management Fee Expense, Gross</link:label>
    <link:label id="lab_efc_ManagementFeeExpenseGross_documentation_en-US" xlink:label="lab_efc_ManagementFeeExpenseGross" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Management Fee Expense, Gross</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ManagementFeeExpenseGross" xlink:href="efc-20201231.xsd#efc_ManagementFeeExpenseGross"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ManagementFeeExpenseGross" xlink:to="lab_efc_ManagementFeeExpenseGross" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancingReceivables1To29DaysPastDueMember_f6290217-b3d6-4fea-afdd-f8aa26b1b4b4_terseLabel_en-US" xlink:label="lab_us-gaap_FinancingReceivables1To29DaysPastDueMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Asset, 1 to 29 Days Past Due [Member]</link:label>
    <link:label id="lab_us-gaap_FinancingReceivables1To29DaysPastDueMember_label_en-US" xlink:label="lab_us-gaap_FinancingReceivables1To29DaysPastDueMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Asset, 1 to 29 Days Past Due [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancingReceivables1To29DaysPastDueMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancingReceivables1To29DaysPastDueMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancingReceivables1To29DaysPastDueMember" xlink:to="lab_us-gaap_FinancingReceivables1To29DaysPastDueMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OffsettingOfAssetsAndLiabilitiesLineItems_aa01d52d-b839-4019-8673-ac500210031e_terseLabel_en-US" xlink:label="lab_efc_OffsettingOfAssetsAndLiabilitiesLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Offsetting of Assets and Liabilities [Line Items]</link:label>
    <link:label id="lab_efc_OffsettingOfAssetsAndLiabilitiesLineItems_label_en-US" xlink:label="lab_efc_OffsettingOfAssetsAndLiabilitiesLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Offsetting of Assets and Liabilities [Line Items]</link:label>
    <link:label id="lab_efc_OffsettingOfAssetsAndLiabilitiesLineItems_documentation_en-US" xlink:label="lab_efc_OffsettingOfAssetsAndLiabilitiesLineItems" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Line Items] for Offsetting of Assets and Liabilities [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OffsettingOfAssetsAndLiabilitiesLineItems" xlink:href="efc-20201231.xsd#efc_OffsettingOfAssetsAndLiabilitiesLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OffsettingOfAssetsAndLiabilitiesLineItems" xlink:to="lab_efc_OffsettingOfAssetsAndLiabilitiesLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StockholdersEquityPeriodIncreaseDecrease_3c4b985f-b29c-4509-aab8-306843f8e62e_terseLabel_en-US" xlink:label="lab_us-gaap_StockholdersEquityPeriodIncreaseDecrease" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stockholders' Equity, Period Increase (Decrease)</link:label>
    <link:label id="lab_us-gaap_StockholdersEquityPeriodIncreaseDecrease_label_en-US" xlink:label="lab_us-gaap_StockholdersEquityPeriodIncreaseDecrease" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stockholders' Equity, Period Increase (Decrease)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StockholdersEquityPeriodIncreaseDecrease" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockholdersEquityPeriodIncreaseDecrease"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StockholdersEquityPeriodIncreaseDecrease" xlink:to="lab_us-gaap_StockholdersEquityPeriodIncreaseDecrease" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Investmentestimatedremaininglife_48b04c72-26e4-46f9-b654-99a1c84cb20d_terseLabel_en-US" xlink:label="lab_efc_Investmentestimatedremaininglife" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Weighted average life</link:label>
    <link:label id="lab_efc_Investmentestimatedremaininglife_label_en-US" xlink:label="lab_efc_Investmentestimatedremaininglife" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment, estimated remaining life</link:label>
    <link:label id="lab_efc_Investmentestimatedremaininglife_documentation_en-US" xlink:label="lab_efc_Investmentestimatedremaininglife" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment, estimated remaining life</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Investmentestimatedremaininglife" xlink:href="efc-20201231.xsd#efc_Investmentestimatedremaininglife"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Investmentestimatedremaininglife" xlink:to="lab_efc_Investmentestimatedremaininglife" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncreaseDecreaseInAccountsPayableAndAccruedLiabilities_c64a14cb-28b1-4eaa-98cf-523e4cb396a8_terseLabel_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInAccountsPayableAndAccruedLiabilities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Accounts payable and accrued expenses</link:label>
    <link:label id="lab_us-gaap_IncreaseDecreaseInAccountsPayableAndAccruedLiabilities_label_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInAccountsPayableAndAccruedLiabilities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (Decrease) in Accounts Payable and Accrued Liabilities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncreaseDecreaseInAccountsPayableAndAccruedLiabilities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncreaseDecreaseInAccountsPayableAndAccruedLiabilities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncreaseDecreaseInAccountsPayableAndAccruedLiabilities" xlink:to="lab_us-gaap_IncreaseDecreaseInAccountsPayableAndAccruedLiabilities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Delinquentloanschargedoff_83567425-888f-4a0b-9601-4ca9307c2abc_terseLabel_en-US" xlink:label="lab_efc_Delinquentloanschargedoff" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Delinquent loans, charged off</link:label>
    <link:label id="lab_efc_Delinquentloanschargedoff_label_en-US" xlink:label="lab_efc_Delinquentloanschargedoff" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Delinquent loans, charged off</link:label>
    <link:label id="lab_efc_Delinquentloanschargedoff_documentation_en-US" xlink:label="lab_efc_Delinquentloanschargedoff" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount of unpaid principal balance that have been charged off during the period. Loans are charged-off when they become 120 days delinquent.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Delinquentloanschargedoff" xlink:href="efc-20201231.xsd#efc_Delinquentloanschargedoff"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Delinquentloanschargedoff" xlink:to="lab_efc_Delinquentloanschargedoff" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityVoluntaryFilers_672b58fb-f89b-4b62-984a-3375658b868b_terseLabel_en-US" xlink:label="lab_dei_EntityVoluntaryFilers" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Voluntary Filers</link:label>
    <link:label id="lab_dei_EntityVoluntaryFilers_label_en-US" xlink:label="lab_dei_EntityVoluntaryFilers" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Voluntary Filers</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityVoluntaryFilers" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityVoluntaryFilers"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityVoluntaryFilers" xlink:to="lab_dei_EntityVoluntaryFilers" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MoneyMarketFundsAtCarryingValue_14dde4e3-da7a-4605-83a6-38cc1ef5f757_terseLabel_en-US" xlink:label="lab_us-gaap_MoneyMarketFundsAtCarryingValue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Money Market Funds, at Carrying Value</link:label>
    <link:label id="lab_us-gaap_MoneyMarketFundsAtCarryingValue_label_en-US" xlink:label="lab_us-gaap_MoneyMarketFundsAtCarryingValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Money Market Funds, at Carrying Value</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MoneyMarketFundsAtCarryingValue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MoneyMarketFundsAtCarryingValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MoneyMarketFundsAtCarryingValue" xlink:to="lab_us-gaap_MoneyMarketFundsAtCarryingValue" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansOnRealEstatePrincipalAmountOfDelinquentLoans_4ddee5ba-0263-4f61-a86a-5b3a188568ae_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstatePrincipalAmountOfDelinquentLoans" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Principal Amount of Delinquent Loans</link:label>
    <link:label id="lab_srt_MortgageLoansOnRealEstatePrincipalAmountOfDelinquentLoans_label_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstatePrincipalAmountOfDelinquentLoans" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Principal Amount of Delinquent Loans</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansOnRealEstatePrincipalAmountOfDelinquentLoans" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansOnRealEstatePrincipalAmountOfDelinquentLoans"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansOnRealEstatePrincipalAmountOfDelinquentLoans" xlink:to="lab_srt_MortgageLoansOnRealEstatePrincipalAmountOfDelinquentLoans" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecurityOwnedAndPledgedAsCollateralFairValue_17291242-4b57-44fd-b91d-9b02bb1c5203_terseLabel_en-US" xlink:label="lab_us-gaap_SecurityOwnedAndPledgedAsCollateralFairValue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Security Owned and Pledged as Collateral, Fair Value</link:label>
    <link:label id="lab_us-gaap_SecurityOwnedAndPledgedAsCollateralFairValue_label_en-US" xlink:label="lab_us-gaap_SecurityOwnedAndPledgedAsCollateralFairValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Security Owned and Pledged as Collateral, Fair Value</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecurityOwnedAndPledgedAsCollateralFairValue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecurityOwnedAndPledgedAsCollateralFairValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecurityOwnedAndPledgedAsCollateralFairValue" xlink:to="lab_us-gaap_SecurityOwnedAndPledgedAsCollateralFairValue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CreditDefaultSwapsOnCorporateBondsMember_a5898f6f-bf01-4c90-8841-df3751f19c53_terseLabel_en-US" xlink:label="lab_efc_CreditDefaultSwapsOnCorporateBondsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit default swaps on corporate bonds</link:label>
    <link:label id="lab_efc_CreditDefaultSwapsOnCorporateBondsMember_label_en-US" xlink:label="lab_efc_CreditDefaultSwapsOnCorporateBondsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swaps On Corporate Bonds [Member]</link:label>
    <link:label id="lab_efc_CreditDefaultSwapsOnCorporateBondsMember_documentation_en-US" xlink:label="lab_efc_CreditDefaultSwapsOnCorporateBondsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swaps On Corporate Bonds [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CreditDefaultSwapsOnCorporateBondsMember" xlink:href="efc-20201231.xsd#efc_CreditDefaultSwapsOnCorporateBondsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CreditDefaultSwapsOnCorporateBondsMember" xlink:to="lab_efc_CreditDefaultSwapsOnCorporateBondsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ReceivablesFromBrokersDealersAndClearingOrganizations_1c17a1c3-e4c1-460f-8087-68a8aff6b852_terseLabel_en-US" xlink:label="lab_us-gaap_ReceivablesFromBrokersDealersAndClearingOrganizations" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Receivables from Brokers-Dealers and Clearing Organizations</link:label>
    <link:label id="lab_us-gaap_ReceivablesFromBrokersDealersAndClearingOrganizations_label_en-US" xlink:label="lab_us-gaap_ReceivablesFromBrokersDealersAndClearingOrganizations" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Receivables from Brokers-Dealers and Clearing Organizations</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ReceivablesFromBrokersDealersAndClearingOrganizations" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ReceivablesFromBrokersDealersAndClearingOrganizations"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ReceivablesFromBrokersDealersAndClearingOrganizations" xlink:to="lab_us-gaap_ReceivablesFromBrokersDealersAndClearingOrganizations" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NoncontrollinginterestpolicyPolicyTextBlock_afb796a7-66d0-4093-94ec-b883dd0ec377_terseLabel_en-US" xlink:label="lab_efc_NoncontrollinginterestpolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling interest</link:label>
    <link:label id="lab_efc_NoncontrollinginterestpolicyPolicyTextBlock_label_en-US" xlink:label="lab_efc_NoncontrollinginterestpolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling interest policy [Policy Text Block]</link:label>
    <link:label id="lab_efc_NoncontrollinginterestpolicyPolicyTextBlock_documentation_en-US" xlink:label="lab_efc_NoncontrollinginterestpolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling interest policy [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NoncontrollinginterestpolicyPolicyTextBlock" xlink:href="efc-20201231.xsd#efc_NoncontrollinginterestpolicyPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NoncontrollinginterestpolicyPolicyTextBlock" xlink:to="lab_efc_NoncontrollinginterestpolicyPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InterestReceivable_418293ec-3595-4896-9084-239984f01168_terseLabel_en-US" xlink:label="lab_us-gaap_InterestReceivable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Receivable</link:label>
    <link:label id="lab_us-gaap_InterestReceivable_label_en-US" xlink:label="lab_us-gaap_InterestReceivable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Receivable</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InterestReceivable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InterestReceivable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InterestReceivable" xlink:to="lab_us-gaap_InterestReceivable" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_CT_6b127610-4f70-47ec-96b1-c618cd362bc7_terseLabel_en-US" xlink:label="lab_stpr_CT" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CONNECTICUT</link:label>
    <link:label id="lab_stpr_CT_label_en-US" xlink:label="lab_stpr_CT" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CONNECTICUT</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_CT" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_CT"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_CT" xlink:to="lab_stpr_CT" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentOwnedUnderlyingFaceAmountAtMarketValue_f87351a2-4213-4ba5-a527-121e6db7af11_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentOwnedUnderlyingFaceAmountAtMarketValue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Current Principal</link:label>
    <link:label id="lab_us-gaap_InvestmentOwnedUnderlyingFaceAmountAtMarketValue_label_en-US" xlink:label="lab_us-gaap_InvestmentOwnedUnderlyingFaceAmountAtMarketValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Owned, Face Amount</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentOwnedUnderlyingFaceAmountAtMarketValue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentOwnedUnderlyingFaceAmountAtMarketValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentOwnedUnderlyingFaceAmountAtMarketValue" xlink:to="lab_us-gaap_InvestmentOwnedUnderlyingFaceAmountAtMarketValue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ExchangeTradedEquityMember_28bead48-52cf-497a-9e36-e9fac63a04f7_terseLabel_en-US" xlink:label="lab_efc_ExchangeTradedEquityMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Exchange Traded Equity [Member]</link:label>
    <link:label id="lab_efc_ExchangeTradedEquityMember_label_en-US" xlink:label="lab_efc_ExchangeTradedEquityMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Exchange Traded Equity [Member]</link:label>
    <link:label id="lab_efc_ExchangeTradedEquityMember_documentation_en-US" xlink:label="lab_efc_ExchangeTradedEquityMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Exchange Traded Equity [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ExchangeTradedEquityMember" xlink:href="efc-20201231.xsd#efc_ExchangeTradedEquityMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ExchangeTradedEquityMember" xlink:to="lab_efc_ExchangeTradedEquityMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PurchasingEntityMember_703f8b0d-b73d-4795-b199-7785707a697f_terseLabel_en-US" xlink:label="lab_efc_PurchasingEntityMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchasing Entity</link:label>
    <link:label id="lab_efc_PurchasingEntityMember_label_en-US" xlink:label="lab_efc_PurchasingEntityMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchasing Entity [Member]</link:label>
    <link:label id="lab_efc_PurchasingEntityMember_documentation_en-US" xlink:label="lab_efc_PurchasingEntityMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Purchasing Entity [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PurchasingEntityMember" xlink:href="efc-20201231.xsd#efc_PurchasingEntityMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PurchasingEntityMember" xlink:to="lab_efc_PurchasingEntityMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_UnrealizedGainLossOnInvestments_458d6f74-1886-49aa-8b69-22e5efcd6aa7_terseLabel_en-US" xlink:label="lab_us-gaap_UnrealizedGainLossOnInvestments" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments</link:label>
    <link:label id="lab_us-gaap_UnrealizedGainLossOnInvestments_label_en-US" xlink:label="lab_us-gaap_UnrealizedGainLossOnInvestments" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unrealized Gain (Loss) on Investments</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_UnrealizedGainLossOnInvestments" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_UnrealizedGainLossOnInvestments"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_UnrealizedGainLossOnInvestments" xlink:to="lab_us-gaap_UnrealizedGainLossOnInvestments" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ParentMember_ec0c51b9-1f4d-4ded-aeba-80e5a925ebe5_terseLabel_en-US" xlink:label="lab_us-gaap_ParentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Stockholders' Equity</link:label>
    <link:label id="lab_us-gaap_ParentMember_label_en-US" xlink:label="lab_us-gaap_ParentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Parent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ParentMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ParentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ParentMember" xlink:to="lab_us-gaap_ParentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LoansPurchasedUnderCommitmentAgreement_ba7d7d58-74ed-4256-ae0d-dd4a7eec5d3e_terseLabel_en-US" xlink:label="lab_efc_LoansPurchasedUnderCommitmentAgreement" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans purchased under commitment agreement</link:label>
    <link:label id="lab_efc_LoansPurchasedUnderCommitmentAgreement_label_en-US" xlink:label="lab_efc_LoansPurchasedUnderCommitmentAgreement" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans purchased under commitment agreement</link:label>
    <link:label id="lab_efc_LoansPurchasedUnderCommitmentAgreement_documentation_en-US" xlink:label="lab_efc_LoansPurchasedUnderCommitmentAgreement" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans purchased under commitment agreement</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LoansPurchasedUnderCommitmentAgreement" xlink:href="efc-20201231.xsd#efc_LoansPurchasedUnderCommitmentAgreement"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LoansPurchasedUnderCommitmentAgreement" xlink:to="lab_efc_LoansPurchasedUnderCommitmentAgreement" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AdditionalPaidInCapitalMember_f39ced60-2911-49d1-8ddd-b76e6ca5701e_terseLabel_en-US" xlink:label="lab_us-gaap_AdditionalPaidInCapitalMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Additional Paid-in Capital</link:label>
    <link:label id="lab_us-gaap_AdditionalPaidInCapitalMember_label_en-US" xlink:label="lab_us-gaap_AdditionalPaidInCapitalMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Additional Paid-in Capital [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AdditionalPaidInCapitalMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AdditionalPaidInCapitalMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AdditionalPaidInCapitalMember" xlink:to="lab_us-gaap_AdditionalPaidInCapitalMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MortgageLoansOnRealEstate_b58ccd27-4eae-47ab-b0be-72eb731122a3_periodStartLabel_en-US" xlink:label="lab_us-gaap_MortgageLoansOnRealEstate" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Beginning Balance</link:label>
    <link:label id="lab_us-gaap_MortgageLoansOnRealEstate_937f50a9-6015-4db9-83c5-dbcbfb377d80_periodEndLabel_en-US" xlink:label="lab_us-gaap_MortgageLoansOnRealEstate" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Ending Balance</link:label>
    <link:label id="lab_us-gaap_MortgageLoansOnRealEstate_label_en-US" xlink:label="lab_us-gaap_MortgageLoansOnRealEstate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MortgageLoansOnRealEstate" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MortgageLoansOnRealEstate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MortgageLoansOnRealEstate" xlink:to="lab_us-gaap_MortgageLoansOnRealEstate" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FlowpurchaseandsaleagreementMember_f78c636b-6d38-4197-b8e4-001317e43385_terseLabel_en-US" xlink:label="lab_efc_FlowpurchaseandsaleagreementMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Flow consumer loan purchase and sale agreement</link:label>
    <link:label id="lab_efc_FlowpurchaseandsaleagreementMember_label_en-US" xlink:label="lab_efc_FlowpurchaseandsaleagreementMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Flow purchase and sale agreement [Member]</link:label>
    <link:label id="lab_efc_FlowpurchaseandsaleagreementMember_documentation_en-US" xlink:label="lab_efc_FlowpurchaseandsaleagreementMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Flow purchase and sale agreement [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FlowpurchaseandsaleagreementMember" xlink:href="efc-20201231.xsd#efc_FlowpurchaseandsaleagreementMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FlowpurchaseandsaleagreementMember" xlink:to="lab_efc_FlowpurchaseandsaleagreementMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CLOISecuritizationMember_7d442405-2bdd-4c4f-a485-aa5564c04228_terseLabel_en-US" xlink:label="lab_efc_CLOISecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLO I Securitization [Member]</link:label>
    <link:label id="lab_efc_CLOISecuritizationMember_label_en-US" xlink:label="lab_efc_CLOISecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLO I Securitization [Member]</link:label>
    <link:label id="lab_efc_CLOISecuritizationMember_documentation_en-US" xlink:label="lab_efc_CLOISecuritizationMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLO I Securitization (CLO transaction June 2017) [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CLOISecuritizationMember" xlink:href="efc-20201231.xsd#efc_CLOISecuritizationMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CLOISecuritizationMember" xlink:to="lab_efc_CLOISecuritizationMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_TextBlockAbstract_label_en-US" xlink:label="lab_us-gaap_TextBlockAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Text Block [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_TextBlockAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_TextBlockAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_TextBlockAbstract" xlink:to="lab_us-gaap_TextBlockAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DeferredTaxAssetsOperatingLossCarryforwards_b7de4b6c-618d-4720-9b73-e0738b32f7b8_terseLabel_en-US" xlink:label="lab_us-gaap_DeferredTaxAssetsOperatingLossCarryforwards" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Tax Assets, Operating Loss Carryforwards</link:label>
    <link:label id="lab_us-gaap_DeferredTaxAssetsOperatingLossCarryforwards_label_en-US" xlink:label="lab_us-gaap_DeferredTaxAssetsOperatingLossCarryforwards" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Tax Assets, Operating Loss Carryforwards</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DeferredTaxAssetsOperatingLossCarryforwards" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DeferredTaxAssetsOperatingLossCarryforwards"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DeferredTaxAssetsOperatingLossCarryforwards" xlink:to="lab_us-gaap_DeferredTaxAssetsOperatingLossCarryforwards" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodValue_5ea569f1-387e-437c-9928-6c883d6a4c11_negatedLabel_en-US" xlink:label="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodValue" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase of shares of common stock</link:label>
    <link:label id="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodValue_b305abee-fb05-44ea-bdaf-b081bf17103a_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodValue" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total cost</link:label>
    <link:label id="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodValue_label_en-US" xlink:label="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stock Repurchased and Retired During Period, Value</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StockRepurchasedAndRetiredDuringPeriodValue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockRepurchasedAndRetiredDuringPeriodValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StockRepurchasedAndRetiredDuringPeriodValue" xlink:to="lab_us-gaap_StockRepurchasedAndRetiredDuringPeriodValue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InvestmentFairValueMeasurementInputRecoveryTimeline_fefb3359-66ac-402b-8bd6-b7b3fb2520bd_terseLabel_en-US" xlink:label="lab_efc_InvestmentFairValueMeasurementInputRecoveryTimeline" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment, measurement input, recovery time</link:label>
    <link:label id="lab_efc_InvestmentFairValueMeasurementInputRecoveryTimeline_label_en-US" xlink:label="lab_efc_InvestmentFairValueMeasurementInputRecoveryTimeline" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Fair Value Measurement Input, Recovery Timeline</link:label>
    <link:label id="lab_efc_InvestmentFairValueMeasurementInputRecoveryTimeline_documentation_en-US" xlink:label="lab_efc_InvestmentFairValueMeasurementInputRecoveryTimeline" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Fair Value Measurement Input, Recovery Timeline</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InvestmentFairValueMeasurementInputRecoveryTimeline" xlink:href="efc-20201231.xsd#efc_InvestmentFairValueMeasurementInputRecoveryTimeline"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InvestmentFairValueMeasurementInputRecoveryTimeline" xlink:to="lab_efc_InvestmentFairValueMeasurementInputRecoveryTimeline" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ShareBasedCompensationOptionAndIncentivePlansPolicy_95feca6a-c577-4814-ad3c-be4655141dc1_terseLabel_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationOptionAndIncentivePlansPolicy" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">LTIP Units</link:label>
    <link:label id="lab_us-gaap_ShareBasedCompensationOptionAndIncentivePlansPolicy_label_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationOptionAndIncentivePlansPolicy" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share-based Payment Arrangement [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ShareBasedCompensationOptionAndIncentivePlansPolicy" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ShareBasedCompensationOptionAndIncentivePlansPolicy"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ShareBasedCompensationOptionAndIncentivePlansPolicy" xlink:to="lab_us-gaap_ShareBasedCompensationOptionAndIncentivePlansPolicy" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DerivativesMaturityDate_7dfdf660-b5e6-4577-a40d-ec0405e7797c_terseLabel_en-US" xlink:label="lab_efc_DerivativesMaturityDate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivatives Maturity Date</link:label>
    <link:label id="lab_efc_DerivativesMaturityDate_label_en-US" xlink:label="lab_efc_DerivativesMaturityDate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivatives Maturity Date</link:label>
    <link:label id="lab_efc_DerivativesMaturityDate_documentation_en-US" xlink:label="lab_efc_DerivativesMaturityDate" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity date of derivative contract.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DerivativesMaturityDate" xlink:href="efc-20201231.xsd#efc_DerivativesMaturityDate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DerivativesMaturityDate" xlink:to="lab_efc_DerivativesMaturityDate" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SubsidiaryOrEquityMethodInvesteeSaleOfStockBySubsidiaryOrEquityInvesteeTable_f38139bc-0025-4524-a03b-c9bf2afd447d_terseLabel_en-US" xlink:label="lab_us-gaap_SubsidiaryOrEquityMethodInvesteeSaleOfStockBySubsidiaryOrEquityInvesteeTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsidiary or Equity Method Investee, Sale of Stock by Subsidiary or Equity Investee [Table]</link:label>
    <link:label id="lab_us-gaap_SubsidiaryOrEquityMethodInvesteeSaleOfStockBySubsidiaryOrEquityInvesteeTable_label_en-US" xlink:label="lab_us-gaap_SubsidiaryOrEquityMethodInvesteeSaleOfStockBySubsidiaryOrEquityInvesteeTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Subsidiary or Equity Method Investee, Sale of Stock by Subsidiary or Equity Investee [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SubsidiaryOrEquityMethodInvesteeSaleOfStockBySubsidiaryOrEquityInvesteeTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SubsidiaryOrEquityMethodInvesteeSaleOfStockBySubsidiaryOrEquityInvesteeTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SubsidiaryOrEquityMethodInvesteeSaleOfStockBySubsidiaryOrEquityInvesteeTable" xlink:to="lab_us-gaap_SubsidiaryOrEquityMethodInvesteeSaleOfStockBySubsidiaryOrEquityInvesteeTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodsAxis_a39e047b-f24b-4b74-9691-5893e7700ce5_terseLabel_en-US" xlink:label="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodsAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase Agreements and Similar Transactions, Maturity Periods [Axis]</link:label>
    <link:label id="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodsAxis_label_en-US" xlink:label="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodsAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase Agreements and Similar Transactions, Maturity Periods [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodsAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodsAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodsAxis" xlink:to="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseMaturityPeriodsAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Maturity151to180daysMember_67df3c4a-3805-4b95-9830-ef5b6334620a_terseLabel_en-US" xlink:label="lab_efc_Maturity151to180daysMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">151 to 180 Days</link:label>
    <link:label id="lab_efc_Maturity151to180daysMember_label_en-US" xlink:label="lab_efc_Maturity151to180daysMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity 151 to 180 days [Member]</link:label>
    <link:label id="lab_efc_Maturity151to180daysMember_documentation_en-US" xlink:label="lab_efc_Maturity151to180daysMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity 151 to 180 days [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Maturity151to180daysMember" xlink:href="efc-20201231.xsd#efc_Maturity151to180daysMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Maturity151to180daysMember" xlink:to="lab_efc_Maturity151to180daysMember" xlink:type="arc" order="1"/>
    <link:label id="lab_stpr_CO_82f5a87c-983e-4617-8993-84ade989b5f7_terseLabel_en-US" xlink:label="lab_stpr_CO" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Colorado [Member]</link:label>
    <link:label id="lab_stpr_CO_label_en-US" xlink:label="lab_stpr_CO" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">COLORADO</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_stpr_CO" xlink:href="https://xbrl.sec.gov/stpr/2018/stpr-2018-01-31.xsd#stpr_CO"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_stpr_CO" xlink:to="lab_stpr_CO" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueDisclosureItemAmountsDomain_1f2d50ce-9175-4f0c-b07c-22d215d658da_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueDisclosureItemAmountsDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Measurement [Domain]</link:label>
    <link:label id="lab_us-gaap_FairValueDisclosureItemAmountsDomain_label_en-US" xlink:label="lab_us-gaap_FairValueDisclosureItemAmountsDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Measurement [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueDisclosureItemAmountsDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueDisclosureItemAmountsDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueDisclosureItemAmountsDomain" xlink:to="lab_us-gaap_FairValueDisclosureItemAmountsDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InterestIncomeOperating_21eb2e38-620c-4765-88ee-045309d93cd5_terseLabel_en-US" xlink:label="lab_us-gaap_InterestIncomeOperating" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest income</link:label>
    <link:label id="lab_us-gaap_InterestIncomeOperating_6435ca87-ea10-49ea-92ec-2a81be5994be_totalLabel_en-US" xlink:label="lab_us-gaap_InterestIncomeOperating" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Income</link:label>
    <link:label id="lab_us-gaap_InterestIncomeOperating_label_en-US" xlink:label="lab_us-gaap_InterestIncomeOperating" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Income, Operating</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InterestIncomeOperating" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InterestIncomeOperating"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InterestIncomeOperating" xlink:to="lab_us-gaap_InterestIncomeOperating" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtInstrumentInterestRateStatedPercentage_489128e2-8cb1-4183-b82d-0feb645265bc_terseLabel_en-US" xlink:label="lab_us-gaap_DebtInstrumentInterestRateStatedPercentage" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Instrument, Interest Rate, Stated Percentage</link:label>
    <link:label id="lab_us-gaap_DebtInstrumentInterestRateStatedPercentage_label_en-US" xlink:label="lab_us-gaap_DebtInstrumentInterestRateStatedPercentage" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Instrument, Interest Rate, Stated Percentage</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtInstrumentInterestRateStatedPercentage" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtInstrumentInterestRateStatedPercentage"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtInstrumentInterestRateStatedPercentage" xlink:to="lab_us-gaap_DebtInstrumentInterestRateStatedPercentage" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ValuationTechniqueDomain_502c7c6f-2be0-498b-abb9-261662aed6d6_terseLabel_en-US" xlink:label="lab_us-gaap_ValuationTechniqueDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Valuation Approach and Technique [Domain]</link:label>
    <link:label id="lab_us-gaap_ValuationTechniqueDomain_label_en-US" xlink:label="lab_us-gaap_ValuationTechniqueDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Valuation Approach and Technique [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ValuationTechniqueDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ValuationTechniqueDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ValuationTechniqueDomain" xlink:to="lab_us-gaap_ValuationTechniqueDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearTwo_b95e60f0-3425-4d74-a8ef-5d171ac83c92_terseLabel_en-US" xlink:label="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearTwo" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2019 Scheduled Repayment of Principal</link:label>
    <link:label id="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearTwo_label_en-US" xlink:label="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearTwo" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Maturities, Repayments of Principal in Year Two</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearTwo" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearTwo"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearTwo" xlink:to="lab_us-gaap_LongTermDebtMaturitiesRepaymentsOfPrincipalInYearTwo" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CreditRiskContractMember_c6791b38-ab82-48dd-9ebe-bbc7e065505a_terseLabel_en-US" xlink:label="lab_us-gaap_CreditRiskContractMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Risk [Member]</link:label>
    <link:label id="lab_us-gaap_CreditRiskContractMember_label_en-US" xlink:label="lab_us-gaap_CreditRiskContractMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Risk Contract [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CreditRiskContractMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CreditRiskContractMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CreditRiskContractMember" xlink:to="lab_us-gaap_CreditRiskContractMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween100000And1249999Member_80b2aee6-0dfb-4e97-a2c8-ab6e0154f7cc_terseLabel_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween100000And1249999Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $1,00,000 and $1,249,999</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween100000And1249999Member_label_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween100000And1249999Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $1,00,000 and $1,249,999 [Member]</link:label>
    <link:label id="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween100000And1249999Member_documentation_en-US" xlink:label="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween100000And1249999Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans between $1,00,000 and $1,249,999</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween100000And1249999Member" xlink:href="efc-20201231.xsd#efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween100000And1249999Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween100000And1249999Member" xlink:to="lab_efc_SECSchedule1229RealEstateCompaniesInvestmentInMortgageLoansBetween100000And1249999Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleOfPercentageOfTotalCollateralOnReverseRepurchaseAgreementsTableTextBlock_85a295b6-6dbc-477f-8d75-1daec1ed84a5_terseLabel_en-US" xlink:label="lab_efc_ScheduleOfPercentageOfTotalCollateralOnReverseRepurchaseAgreementsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Percentage Of Total Collateral On Reverse Repurchase Agreements [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleOfPercentageOfTotalCollateralOnReverseRepurchaseAgreementsTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleOfPercentageOfTotalCollateralOnReverseRepurchaseAgreementsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule Of Percentage Of Total Collateral On Reverse Repurchase Agreements [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleOfPercentageOfTotalCollateralOnReverseRepurchaseAgreementsTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleOfPercentageOfTotalCollateralOnReverseRepurchaseAgreementsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Tabular disclosure of the Company's percentage of total collateral on reverse repurchase agreements held by a counterparty that is above a specified concentration risk threshold.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleOfPercentageOfTotalCollateralOnReverseRepurchaseAgreementsTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleOfPercentageOfTotalCollateralOnReverseRepurchaseAgreementsTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleOfPercentageOfTotalCollateralOnReverseRepurchaseAgreementsTableTextBlock" xlink:to="lab_efc_ScheduleOfPercentageOfTotalCollateralOnReverseRepurchaseAgreementsTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Returnofcapital_667024d4-4156-4d3e-b655-1bb223e1a607_terseLabel_en-US" xlink:label="lab_efc_Returnofcapital" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Return of capital</link:label>
    <link:label id="lab_efc_Returnofcapital_label_en-US" xlink:label="lab_efc_Returnofcapital" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Return of capital</link:label>
    <link:label id="lab_efc_Returnofcapital_documentation_en-US" xlink:label="lab_efc_Returnofcapital" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The percentage of dividends which are classified as return of capital for tax purposes.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Returnofcapital" xlink:href="efc-20201231.xsd#efc_Returnofcapital"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Returnofcapital" xlink:to="lab_efc_Returnofcapital" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_UnpaidPrincipalBalanceImpairedLoans_d2605569-1965-4763-9434-ba5a71fe113b_terseLabel_en-US" xlink:label="lab_efc_UnpaidPrincipalBalanceImpairedLoans" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unpaid Principal Balance, Impaired Loans</link:label>
    <link:label id="lab_efc_UnpaidPrincipalBalanceImpairedLoans_label_en-US" xlink:label="lab_efc_UnpaidPrincipalBalanceImpairedLoans" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unpaid Principal Balance, Impaired Loans</link:label>
    <link:label id="lab_efc_UnpaidPrincipalBalanceImpairedLoans_documentation_en-US" xlink:label="lab_efc_UnpaidPrincipalBalanceImpairedLoans" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unpaid principal balance of loans that incurred realized losses from impairment charges.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_UnpaidPrincipalBalanceImpairedLoans" xlink:href="efc-20201231.xsd#efc_UnpaidPrincipalBalanceImpairedLoans"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_UnpaidPrincipalBalanceImpairedLoans" xlink:to="lab_efc_UnpaidPrincipalBalanceImpairedLoans" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearTwoPointFourNinePercentMember_afbc4f4e-440a-4e72-aceb-aa4cf43b7abe_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearTwoPointFourNinePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Two Point Four Nine Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearTwoPointFourNinePercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearTwoPointFourNinePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Two Point Four Nine Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearTwoPointFourNinePercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearTwoPointFourNinePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Two Point Four Nine Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearTwoPointFourNinePercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationPoolsThirtyYearTwoPointFourNinePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearTwoPointFourNinePercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearTwoPointFourNinePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PrincipalamountofnotespurchasedfromrelatedpartyCLOsecuritization_af729b0d-4684-46e6-bbea-36f1ded72b51_terseLabel_en-US" xlink:label="lab_efc_PrincipalamountofnotespurchasedfromrelatedpartyCLOsecuritization" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Face Amount of Notes Initially Purchased</link:label>
    <link:label id="lab_efc_PrincipalamountofnotespurchasedfromrelatedpartyCLOsecuritization_label_en-US" xlink:label="lab_efc_PrincipalamountofnotespurchasedfromrelatedpartyCLOsecuritization" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal amount of notes purchased from related party CLO securitization</link:label>
    <link:label id="lab_efc_PrincipalamountofnotespurchasedfromrelatedpartyCLOsecuritization_documentation_en-US" xlink:label="lab_efc_PrincipalamountofnotespurchasedfromrelatedpartyCLOsecuritization" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Principal amount of notes purchased from related party CLO securitization</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PrincipalamountofnotespurchasedfromrelatedpartyCLOsecuritization" xlink:href="efc-20201231.xsd#efc_PrincipalamountofnotespurchasedfromrelatedpartyCLOsecuritization"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PrincipalamountofnotespurchasedfromrelatedpartyCLOsecuritization" xlink:to="lab_efc_PrincipalamountofnotespurchasedfromrelatedpartyCLOsecuritization" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Totalsecuredborrowings_62a9137c-3812-46ef-a488-7b269d34beb2_terseLabel_en-US" xlink:label="lab_efc_Totalsecuredborrowings" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total secured borrowings</link:label>
    <link:label id="lab_efc_Totalsecuredborrowings_label_en-US" xlink:label="lab_efc_Totalsecuredborrowings" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total secured borrowings</link:label>
    <link:label id="lab_efc_Totalsecuredborrowings_documentation_en-US" xlink:label="lab_efc_Totalsecuredborrowings" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total secured borrowings under reverse repurchase agreements (securities sold under agreements to repurchase) and other secured borrowing transactions.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Totalsecuredborrowings" xlink:href="efc-20201231.xsd#efc_Totalsecuredborrowings"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Totalsecuredborrowings" xlink:to="lab_efc_Totalsecuredborrowings" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_VestMarch72019Member_983fb27f-8546-4a4e-a2c1-d9d68849c469_terseLabel_en-US" xlink:label="lab_efc_VestMarch72019Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest March 7, 2019 [Member]</link:label>
    <link:label id="lab_efc_VestMarch72019Member_label_en-US" xlink:label="lab_efc_VestMarch72019Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest March 7, 2019 [Member]</link:label>
    <link:label id="lab_efc_VestMarch72019Member_documentation_en-US" xlink:label="lab_efc_VestMarch72019Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest March 7, 2019 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_VestMarch72019Member" xlink:href="efc-20201231.xsd#efc_VestMarch72019Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_VestMarch72019Member" xlink:to="lab_efc_VestMarch72019Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2035Member_8f28461b-0788-4005-90ca-9138117666e5_terseLabel_en-US" xlink:label="lab_efc_A2035Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2035</link:label>
    <link:label id="lab_efc_A2035Member_label_en-US" xlink:label="lab_efc_A2035Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2035 [Member]</link:label>
    <link:label id="lab_efc_A2035Member_documentation_en-US" xlink:label="lab_efc_A2035Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2035</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2035Member" xlink:href="efc-20201231.xsd#efc_A2035Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2035Member" xlink:to="lab_efc_A2035Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LiabilitiesFairValueDisclosure_bd6ffd20-a286-4bab-9c40-20c2a7301cb7_negatedTotalLabel_en-US" xlink:label="lab_us-gaap_LiabilitiesFairValueDisclosure" xlink:role="http://www.xbrl.org/2009/role/negatedTotalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total liabilities</link:label>
    <link:label id="lab_us-gaap_LiabilitiesFairValueDisclosure_label_en-US" xlink:label="lab_us-gaap_LiabilitiesFairValueDisclosure" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial and Nonfinancial Liabilities, Fair Value Disclosure</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LiabilitiesFairValueDisclosure" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LiabilitiesFairValueDisclosure"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LiabilitiesFairValueDisclosure" xlink:to="lab_us-gaap_LiabilitiesFairValueDisclosure" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellCollateralObligationToReturnCash_432e0ca8-3224-4bfc-8b42-47094f404341_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellCollateralObligationToReturnCash" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash Collateral (Received) Pledged</link:label>
    <link:label id="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellCollateralObligationToReturnCash_label_en-US" xlink:label="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellCollateralObligationToReturnCash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Purchased under Agreements to Resell, Collateral, Obligation to Return Cash</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesPurchasedUnderAgreementsToResellCollateralObligationToReturnCash" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesPurchasedUnderAgreementsToResellCollateralObligationToReturnCash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesPurchasedUnderAgreementsToResellCollateralObligationToReturnCash" xlink:to="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellCollateralObligationToReturnCash" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MeasurementInputPrepaymentRateMember_4483d6d6-5707-48b0-bba8-ab60d86accaa_terseLabel_en-US" xlink:label="lab_us-gaap_MeasurementInputPrepaymentRateMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, Prepayment Rate</link:label>
    <link:label id="lab_us-gaap_MeasurementInputPrepaymentRateMember_label_en-US" xlink:label="lab_us-gaap_MeasurementInputPrepaymentRateMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Input, Prepayment Rate [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MeasurementInputPrepaymentRateMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MeasurementInputPrepaymentRateMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MeasurementInputPrepaymentRateMember" xlink:to="lab_us-gaap_MeasurementInputPrepaymentRateMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearSixPercentMember_4b3c67d2-bc75-42b3-9413-1257756a5200_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearSixPercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Six Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearSixPercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearSixPercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Six Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearSixPercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearSixPercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Six Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearSixPercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationPoolsThirtyYearSixPercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearSixPercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearSixPercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleofResidentialLoanSecuritizationsTableTextBlock_55f2a85d-523e-42fb-b7f9-abc62c15f273_terseLabel_en-US" xlink:label="lab_efc_ScheduleofResidentialLoanSecuritizationsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Residential Loan Securitizations</link:label>
    <link:label id="lab_efc_ScheduleofResidentialLoanSecuritizationsTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleofResidentialLoanSecuritizationsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Residential Loan Securitizations [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleofResidentialLoanSecuritizationsTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleofResidentialLoanSecuritizationsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Residential Loan Securitizations [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleofResidentialLoanSecuritizationsTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleofResidentialLoanSecuritizationsTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleofResidentialLoanSecuritizationsTableTextBlock" xlink:to="lab_efc_ScheduleofResidentialLoanSecuritizationsTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationFivePointFivePercentMember_d160e8aa-fab9-4ada-b34e-927baa2bfab7_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationFivePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Five Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationFivePointFivePercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationFivePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Five Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationFivePointFivePercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationFivePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Five Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationFivePointFivePercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationFivePointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationFivePointFivePercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationFivePointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseAxis_87fdfb7f-9bf6-4b61-8def-11f4e2cc0ba4_terseLabel_en-US" xlink:label="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities or Other Assets Sold under Agreements to Repurchase [Axis]</link:label>
    <link:label id="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseAxis_label_en-US" xlink:label="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities or Other Assets Sold under Agreements to Repurchase [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AssetsSoldUnderAgreementsToRepurchaseAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AssetsSoldUnderAgreementsToRepurchaseAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AssetsSoldUnderAgreementsToRepurchaseAxis" xlink:to="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DividendsPayableAmountPerShare_2f8a9981-e033-4aa2-8f19-23302ed1ef8d_terseLabel_en-US" xlink:label="lab_us-gaap_DividendsPayableAmountPerShare" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividend Per Share (USD per share)</link:label>
    <link:label id="lab_us-gaap_DividendsPayableAmountPerShare_label_en-US" xlink:label="lab_us-gaap_DividendsPayableAmountPerShare" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Dividends Payable, Amount Per Share</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DividendsPayableAmountPerShare" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DividendsPayableAmountPerShare"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DividendsPayableAmountPerShare" xlink:to="lab_us-gaap_DividendsPayableAmountPerShare" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CostofnotespurchasedfromrelatedpartyCLOsecuritization_3b0978b5-c6ef-4e7b-8fc4-032b5c5ca510_terseLabel_en-US" xlink:label="lab_efc_CostofnotespurchasedfromrelatedpartyCLOsecuritization" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Aggregate Purchase Price of Notes Initially Purchased</link:label>
    <link:label id="lab_efc_CostofnotespurchasedfromrelatedpartyCLOsecuritization_label_en-US" xlink:label="lab_efc_CostofnotespurchasedfromrelatedpartyCLOsecuritization" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cost of notes purchased from related party CLO securitization</link:label>
    <link:label id="lab_efc_CostofnotespurchasedfromrelatedpartyCLOsecuritization_documentation_en-US" xlink:label="lab_efc_CostofnotespurchasedfromrelatedpartyCLOsecuritization" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cost of notes purchased from related party CLO securitization</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CostofnotespurchasedfromrelatedpartyCLOsecuritization" xlink:href="efc-20201231.xsd#efc_CostofnotespurchasedfromrelatedpartyCLOsecuritization"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CostofnotespurchasedfromrelatedpartyCLOsecuritization" xlink:to="lab_efc_CostofnotespurchasedfromrelatedpartyCLOsecuritization" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_MortgageLoansOnRealEstateFederalIncomeTaxBasis1_6b88fc49-8f2e-4b63-bbb7-7b862e280987_terseLabel_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateFederalIncomeTaxBasis1" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Federal Income Tax Basis</link:label>
    <link:label id="lab_srt_MortgageLoansOnRealEstateFederalIncomeTaxBasis1_label_en-US" xlink:label="lab_srt_MortgageLoansOnRealEstateFederalIncomeTaxBasis1" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">SEC Schedule, 12-29, Real Estate Companies, Investment in Mortgage Loans on Real Estate, Federal Income Tax Basis</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_MortgageLoansOnRealEstateFederalIncomeTaxBasis1" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_MortgageLoansOnRealEstateFederalIncomeTaxBasis1"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_MortgageLoansOnRealEstateFederalIncomeTaxBasis1" xlink:to="lab_srt_MortgageLoansOnRealEstateFederalIncomeTaxBasis1" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OtherSecuredFinancings_fd69269c-933f-40b9-81ae-2a7d3d309311_terseLabel_en-US" xlink:label="lab_us-gaap_OtherSecuredFinancings" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other secured borrowings</link:label>
    <link:label id="lab_us-gaap_OtherSecuredFinancings_label_en-US" xlink:label="lab_us-gaap_OtherSecuredFinancings" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Secured Financings</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OtherSecuredFinancings" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OtherSecuredFinancings"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OtherSecuredFinancings" xlink:to="lab_us-gaap_OtherSecuredFinancings" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NumberofSharesUnderlyingWarrantContracts_0edb2042-8e45-419f-a9be-70277e494561_terseLabel_en-US" xlink:label="lab_efc_NumberofSharesUnderlyingWarrantContracts" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of Shares Underlying Warrant Contracts</link:label>
    <link:label id="lab_efc_NumberofSharesUnderlyingWarrantContracts_label_en-US" xlink:label="lab_efc_NumberofSharesUnderlyingWarrantContracts" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of Shares Underlying Warrant Contracts</link:label>
    <link:label id="lab_efc_NumberofSharesUnderlyingWarrantContracts_documentation_en-US" xlink:label="lab_efc_NumberofSharesUnderlyingWarrantContracts" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of Shares Underlying Warrant Contracts</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NumberofSharesUnderlyingWarrantContracts" xlink:href="efc-20201231.xsd#efc_NumberofSharesUnderlyingWarrantContracts"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NumberofSharesUnderlyingWarrantContracts" xlink:to="lab_efc_NumberofSharesUnderlyingWarrantContracts" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RepurchaseAndResaleAgreementsPolicy_0c322a08-1fc6-4dee-9fd3-21e8ac928829_verboseLabel_en-US" xlink:label="lab_us-gaap_RepurchaseAndResaleAgreementsPolicy" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase and Resale Agreements</link:label>
    <link:label id="lab_us-gaap_RepurchaseAndResaleAgreementsPolicy_label_en-US" xlink:label="lab_us-gaap_RepurchaseAndResaleAgreementsPolicy" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repurchase and Resale Agreements Policy [Policy Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RepurchaseAndResaleAgreementsPolicy" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RepurchaseAndResaleAgreementsPolicy"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RepurchaseAndResaleAgreementsPolicy" xlink:to="lab_us-gaap_RepurchaseAndResaleAgreementsPolicy" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentCompanyInvestmentIncomeLossPerShare_b10baf24-7a48-4db4-afe2-80545020310f_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentCompanyInvestmentIncomeLossPerShare" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Investment Income</link:label>
    <link:label id="lab_us-gaap_InvestmentCompanyInvestmentIncomeLossPerShare_label_en-US" xlink:label="lab_us-gaap_InvestmentCompanyInvestmentIncomeLossPerShare" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Company, Investment Income (Loss), Per Share</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentCompanyInvestmentIncomeLossPerShare" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentCompanyInvestmentIncomeLossPerShare"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentCompanyInvestmentIncomeLossPerShare" xlink:to="lab_us-gaap_InvestmentCompanyInvestmentIncomeLossPerShare" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePercentMember_114b142c-c155-4e3e-8350-660dc1807663_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellLiability_04ac8bce-1bb5-4247-aff2-ed847ccd56a6_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellLiability" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Purchased under Agreements to Resell, Liability</link:label>
    <link:label id="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellLiability_label_en-US" xlink:label="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellLiability" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Purchased under Agreements to Resell, Liability</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesPurchasedUnderAgreementsToResellLiability" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesPurchasedUnderAgreementsToResellLiability"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesPurchasedUnderAgreementsToResellLiability" xlink:to="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellLiability" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_EntityCommonStockSharesOutstanding_e8ed72ff-d8f2-4f65-8f86-38cfe0301393_terseLabel_en-US" xlink:label="lab_dei_EntityCommonStockSharesOutstanding" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Common Stock, Shares Outstanding</link:label>
    <link:label id="lab_dei_EntityCommonStockSharesOutstanding_label_en-US" xlink:label="lab_dei_EntityCommonStockSharesOutstanding" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Entity Common Stock, Shares Outstanding</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_EntityCommonStockSharesOutstanding" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_EntityCommonStockSharesOutstanding"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_EntityCommonStockSharesOutstanding" xlink:to="lab_dei_EntityCommonStockSharesOutstanding" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ToBeAnnouncedSecuritiesTableTextBlock_96f1bbac-c36c-46d4-9479-b413309ff316_terseLabel_en-US" xlink:label="lab_efc_ToBeAnnouncedSecuritiesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">To Be Announced RMBS</link:label>
    <link:label id="lab_efc_ToBeAnnouncedSecuritiesTableTextBlock_label_en-US" xlink:label="lab_efc_ToBeAnnouncedSecuritiesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">To Be Announced Securities [Table Text Block]</link:label>
    <link:label id="lab_efc_ToBeAnnouncedSecuritiesTableTextBlock_documentation_en-US" xlink:label="lab_efc_ToBeAnnouncedSecuritiesTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Table Text Block] for To Be Announced Securities [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ToBeAnnouncedSecuritiesTableTextBlock" xlink:href="efc-20201231.xsd#efc_ToBeAnnouncedSecuritiesTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ToBeAnnouncedSecuritiesTableTextBlock" xlink:to="lab_efc_ToBeAnnouncedSecuritiesTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Numberofcontracts_02d40864-41f4-4673-a368-4ac7a9ee4ee1_terseLabel_en-US" xlink:label="lab_efc_Numberofcontracts" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of contracts</link:label>
    <link:label id="lab_efc_Numberofcontracts_label_en-US" xlink:label="lab_efc_Numberofcontracts" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of contracts</link:label>
    <link:label id="lab_efc_Numberofcontracts_documentation_en-US" xlink:label="lab_efc_Numberofcontracts" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of future contracts.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Numberofcontracts" xlink:href="efc-20201231.xsd#efc_Numberofcontracts"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Numberofcontracts" xlink:to="lab_efc_Numberofcontracts" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CollateralizedLoanObligationsAndOtherSecuritiesMember_2f7c8dab-9666-4aa5-b004-360059c19bbf_terseLabel_en-US" xlink:label="lab_efc_CollateralizedLoanObligationsAndOtherSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLOs And Other Securities [Member]</link:label>
    <link:label id="lab_efc_CollateralizedLoanObligationsAndOtherSecuritiesMember_label_en-US" xlink:label="lab_efc_CollateralizedLoanObligationsAndOtherSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Collateralized Loan Obligations And Other Securities [Member]</link:label>
    <link:label id="lab_efc_CollateralizedLoanObligationsAndOtherSecuritiesMember_documentation_en-US" xlink:label="lab_efc_CollateralizedLoanObligationsAndOtherSecuritiesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Collateralized Loan Obligations And Other Securities [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CollateralizedLoanObligationsAndOtherSecuritiesMember" xlink:href="efc-20201231.xsd#efc_CollateralizedLoanObligationsAndOtherSecuritiesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CollateralizedLoanObligationsAndOtherSecuritiesMember" xlink:to="lab_efc_CollateralizedLoanObligationsAndOtherSecuritiesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfEquityMethodInvestmentsLineItems_d8b2f1ee-c543-40da-87d2-9aef65a83ee9_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfEquityMethodInvestmentsLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Equity Method Investments [Line Items]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfEquityMethodInvestmentsLineItems_label_en-US" xlink:label="lab_us-gaap_ScheduleOfEquityMethodInvestmentsLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Equity Method Investments [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfEquityMethodInvestmentsLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfEquityMethodInvestmentsLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfEquityMethodInvestmentsLineItems" xlink:to="lab_us-gaap_ScheduleOfEquityMethodInvestmentsLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PercentageOfDuefromBroker_cf4fcc61-6d92-4ea2-b92b-6ea057a3f1a8_terseLabel_en-US" xlink:label="lab_efc_PercentageOfDuefromBroker" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">%&#160;of&#160;Total&#160;Due from Brokers</link:label>
    <link:label id="lab_efc_PercentageOfDuefromBroker_label_en-US" xlink:label="lab_efc_PercentageOfDuefromBroker" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage Of Due from Broker</link:label>
    <link:label id="lab_efc_PercentageOfDuefromBroker_documentation_en-US" xlink:label="lab_efc_PercentageOfDuefromBroker" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage of total amount held by a specific counterparty included in Due from Brokers above a specified concentration risk threshold.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PercentageOfDuefromBroker" xlink:href="efc-20201231.xsd#efc_PercentageOfDuefromBroker"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PercentageOfDuefromBroker" xlink:to="lab_efc_PercentageOfDuefromBroker" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_UpFrontFeesPaidReceivedOnCreditDerivatives_42245117-4e74-4b91-acf4-087cb4597b4c_terseLabel_en-US" xlink:label="lab_efc_UpFrontFeesPaidReceivedOnCreditDerivatives" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total net up-front payments received</link:label>
    <link:label id="lab_efc_UpFrontFeesPaidReceivedOnCreditDerivatives_label_en-US" xlink:label="lab_efc_UpFrontFeesPaidReceivedOnCreditDerivatives" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Up-Front Fees (Paid) Received On Credit Derivatives</link:label>
    <link:label id="lab_efc_UpFrontFeesPaidReceivedOnCreditDerivatives_documentation_en-US" xlink:label="lab_efc_UpFrontFeesPaidReceivedOnCreditDerivatives" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total net upfront fees (paid) received on credit derivatives.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_UpFrontFeesPaidReceivedOnCreditDerivatives" xlink:href="efc-20201231.xsd#efc_UpFrontFeesPaidReceivedOnCreditDerivatives"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_UpFrontFeesPaidReceivedOnCreditDerivatives" xlink:to="lab_efc_UpFrontFeesPaidReceivedOnCreditDerivatives" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EllingtonFinancialOperatingPartnershipLLCMember_ea132d42-59c9-4287-bddc-9604faed4d27_terseLabel_en-US" xlink:label="lab_efc_EllingtonFinancialOperatingPartnershipLLCMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ellington Financial Operating Partnership LLC [Member]</link:label>
    <link:label id="lab_efc_EllingtonFinancialOperatingPartnershipLLCMember_label_en-US" xlink:label="lab_efc_EllingtonFinancialOperatingPartnershipLLCMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ellington Financial Operating Partnership LLC [Member]</link:label>
    <link:label id="lab_efc_EllingtonFinancialOperatingPartnershipLLCMember_documentation_en-US" xlink:label="lab_efc_EllingtonFinancialOperatingPartnershipLLCMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ellington Financial Operating Partnership LLC [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EllingtonFinancialOperatingPartnershipLLCMember" xlink:href="efc-20201231.xsd#efc_EllingtonFinancialOperatingPartnershipLLCMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EllingtonFinancialOperatingPartnershipLLCMember" xlink:to="lab_efc_EllingtonFinancialOperatingPartnershipLLCMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationFourPointFivePercentMember_e020758b-eda3-444d-acd5-a5e419c170b2_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationFourPointFivePercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationFourPointFivePercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Four Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationFourPointFivePercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationFourPointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationFourPointFivePercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationFourPointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentRelatedLiabilities_449e41af-9bbe-488f-ae5e-fc92af6d2756_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentRelatedLiabilities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total investments and financial derivatives</link:label>
    <link:label id="lab_us-gaap_InvestmentRelatedLiabilities_label_en-US" xlink:label="lab_us-gaap_InvestmentRelatedLiabilities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment-related Liabilities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentRelatedLiabilities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentRelatedLiabilities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentRelatedLiabilities" xlink:to="lab_us-gaap_InvestmentRelatedLiabilities" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueByMeasurementFrequencyAxis_b09cd64e-80a7-4ba0-99a9-ea1438acf0b6_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueByMeasurementFrequencyAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Frequency [Axis]</link:label>
    <link:label id="lab_us-gaap_FairValueByMeasurementFrequencyAxis_label_en-US" xlink:label="lab_us-gaap_FairValueByMeasurementFrequencyAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Measurement Frequency [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueByMeasurementFrequencyAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueByMeasurementFrequencyAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueByMeasurementFrequencyAxis" xlink:to="lab_us-gaap_FairValueByMeasurementFrequencyAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilities_a744b9e9-4a07-4782-b270-561ad6fa9bc5_terseLabel_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Liabilities</link:label>
    <link:label id="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilities_label_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Liabilities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilities" xlink:to="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationLiabilities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_DiversifiedMember_04734414-8837-4874-9dd7-7d1e6c08d4a4_terseLabel_en-US" xlink:label="lab_efc_DiversifiedMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Diversified [Member]</link:label>
    <link:label id="lab_efc_DiversifiedMember_label_en-US" xlink:label="lab_efc_DiversifiedMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Diversified [Member]</link:label>
    <link:label id="lab_efc_DiversifiedMember_documentation_en-US" xlink:label="lab_efc_DiversifiedMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Diversified [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_DiversifiedMember" xlink:href="efc-20201231.xsd#efc_DiversifiedMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_DiversifiedMember" xlink:to="lab_efc_DiversifiedMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EarningsPerShareTextBlock_165c49d4-4f63-443b-8ccf-799a9b716937_terseLabel_en-US" xlink:label="lab_us-gaap_EarningsPerShareTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share</link:label>
    <link:label id="lab_us-gaap_EarningsPerShareTextBlock_label_en-US" xlink:label="lab_us-gaap_EarningsPerShareTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Earnings Per Share [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EarningsPerShareTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EarningsPerShareTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EarningsPerShareTextBlock" xlink:to="lab_us-gaap_EarningsPerShareTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestIncome_1ac4ac80-0153-43d1-86b2-d4a0604ae004_terseLabel_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestIncome" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Interest Income</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestIncome_label_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestIncome" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Interest Income</link:label>
    <link:label id="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestIncome_documentation_en-US" xlink:label="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestIncome" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount of interest income reported by equity method investee.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestIncome" xlink:href="efc-20201231.xsd#efc_EquityMethodInvestmentSummarizedFinancialInformationInterestIncome"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestIncome" xlink:to="lab_efc_EquityMethodInvestmentSummarizedFinancialInformationInterestIncome" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentSecondaryCategorizationAxis_15c60ca8-af6d-41a5-81b9-f79b77657a7c_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentSecondaryCategorizationAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Secondary Categorization [Axis]</link:label>
    <link:label id="lab_us-gaap_InvestmentSecondaryCategorizationAxis_label_en-US" xlink:label="lab_us-gaap_InvestmentSecondaryCategorizationAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Secondary Categorization [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentSecondaryCategorizationAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentSecondaryCategorizationAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentSecondaryCategorizationAxis" xlink:to="lab_us-gaap_InvestmentSecondaryCategorizationAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardAwardTypeAndPlanNameDomain_ff92f994-e1b6-4987-a148-c6cf4b6d8f2f_terseLabel_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardAwardTypeAndPlanNameDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Award Type [Domain]</link:label>
    <link:label id="lab_us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardAwardTypeAndPlanNameDomain_label_en-US" xlink:label="lab_us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardAwardTypeAndPlanNameDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Award Type [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardAwardTypeAndPlanNameDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardAwardTypeAndPlanNameDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardAwardTypeAndPlanNameDomain" xlink:to="lab_us-gaap_ShareBasedCompensationArrangementsByShareBasedPaymentAwardAwardTypeAndPlanNameDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_USTreasuryNoteFuturesMember_f5c139da-519d-4b9b-8b42-f9cb5dea1ceb_terseLabel_en-US" xlink:label="lab_efc_USTreasuryNoteFuturesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">US Treasury Note Futures [Member]</link:label>
    <link:label id="lab_efc_USTreasuryNoteFuturesMember_label_en-US" xlink:label="lab_efc_USTreasuryNoteFuturesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">US Treasury Note Futures [Member]</link:label>
    <link:label id="lab_efc_USTreasuryNoteFuturesMember_documentation_en-US" xlink:label="lab_efc_USTreasuryNoteFuturesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">US Treasury Note Futures [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_USTreasuryNoteFuturesMember" xlink:href="efc-20201231.xsd#efc_USTreasuryNoteFuturesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_USTreasuryNoteFuturesMember" xlink:to="lab_efc_USTreasuryNoteFuturesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CurrencyFuturesMember_fb946d4b-f3b9-48b0-8581-18e76a91a4c0_terseLabel_en-US" xlink:label="lab_efc_CurrencyFuturesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Currency futures</link:label>
    <link:label id="lab_efc_CurrencyFuturesMember_label_en-US" xlink:label="lab_efc_CurrencyFuturesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Currency Futures [Member]</link:label>
    <link:label id="lab_efc_CurrencyFuturesMember_documentation_en-US" xlink:label="lab_efc_CurrencyFuturesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Currency Futures [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CurrencyFuturesMember" xlink:href="efc-20201231.xsd#efc_CurrencyFuturesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CurrencyFuturesMember" xlink:to="lab_efc_CurrencyFuturesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NonExchangeTradedCorporateEquityMember_eb16c186-0e1d-4f63-99ec-db1a7b80fbd9_terseLabel_en-US" xlink:label="lab_efc_NonExchangeTradedCorporateEquityMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Corporate Equity [Member]</link:label>
    <link:label id="lab_efc_NonExchangeTradedCorporateEquityMember_label_en-US" xlink:label="lab_efc_NonExchangeTradedCorporateEquityMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Corporate Equity [Member]</link:label>
    <link:label id="lab_efc_NonExchangeTradedCorporateEquityMember_documentation_en-US" xlink:label="lab_efc_NonExchangeTradedCorporateEquityMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-Exchange Traded Corporate Equity [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NonExchangeTradedCorporateEquityMember" xlink:href="efc-20201231.xsd#efc_NonExchangeTradedCorporateEquityMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NonExchangeTradedCorporateEquityMember" xlink:to="lab_efc_NonExchangeTradedCorporateEquityMember" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_OwnershipDomain_2f7fce44-ac00-43ce-a78a-89c338295dce_terseLabel_en-US" xlink:label="lab_srt_OwnershipDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ownership [Domain]</link:label>
    <link:label id="lab_srt_OwnershipDomain_label_en-US" xlink:label="lab_srt_OwnershipDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ownership [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_OwnershipDomain" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_OwnershipDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_OwnershipDomain" xlink:to="lab_srt_OwnershipDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection_197c9550-6dc5-41f1-a991-2b2107bdf3d4_terseLabel_en-US" xlink:label="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Amount</link:label>
    <link:label id="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection_label_en-US" xlink:label="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Purchased under Agreements to Resell, Offset Against Collateral, Net of Not Subject to Master Netting Arrangement, Policy Election</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_SecuritiesPurchasedUnderAgreementsToResellOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_SecuritiesPurchasedUnderAgreementsToResellOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_SecuritiesPurchasedUnderAgreementsToResellOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:to="lab_us-gaap_SecuritiesPurchasedUnderAgreementsToResellOffsetAgainstCollateralNetOfNotSubjectToMasterNettingArrangementPolicyElection" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RelatedPartyTransactionPurchasesFromRelatedParty_01eb86d5-0aa0-442e-a87e-1fc701baa69d_terseLabel_en-US" xlink:label="lab_us-gaap_RelatedPartyTransactionPurchasesFromRelatedParty" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party Transaction, Purchases from Related Party</link:label>
    <link:label id="lab_us-gaap_RelatedPartyTransactionPurchasesFromRelatedParty_label_en-US" xlink:label="lab_us-gaap_RelatedPartyTransactionPurchasesFromRelatedParty" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Related Party Transaction, Purchases from Related Party</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RelatedPartyTransactionPurchasesFromRelatedParty" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RelatedPartyTransactionPurchasesFromRelatedParty"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RelatedPartyTransactionPurchasesFromRelatedParty" xlink:to="lab_us-gaap_RelatedPartyTransactionPurchasesFromRelatedParty" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LoansHeldForSaleMember_71dcb02d-57a1-4497-93b8-9660a1737ba3_terseLabel_en-US" xlink:label="lab_efc_LoansHeldForSaleMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans held-for-sale</link:label>
    <link:label id="lab_efc_LoansHeldForSaleMember_label_en-US" xlink:label="lab_efc_LoansHeldForSaleMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans held-for-sale [Member]</link:label>
    <link:label id="lab_efc_LoansHeldForSaleMember_documentation_en-US" xlink:label="lab_efc_LoansHeldForSaleMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans held-for-sale</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LoansHeldForSaleMember" xlink:href="efc-20201231.xsd#efc_LoansHeldForSaleMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LoansHeldForSaleMember" xlink:to="lab_efc_LoansHeldForSaleMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems_5a9e8726-ce84-4556-a280-a9332bb58c40_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Assets Measured on Recurring Basis, Unobservable Input Reconciliation [Line Items]</link:label>
    <link:label id="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems_label_en-US" xlink:label="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Assets Measured on Recurring Basis, Unobservable Input Reconciliation [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems" xlink:to="lab_us-gaap_FairValueAssetsMeasuredOnRecurringBasisUnobservableInputReconciliationLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_GainLossOnSaleOfProperties_69518e4c-46c6-42d2-882c-96f7f9a27318_terseLabel_en-US" xlink:label="lab_us-gaap_GainLossOnSaleOfProperties" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized Gain (Loss) on Sale of Properties</link:label>
    <link:label id="lab_us-gaap_GainLossOnSaleOfProperties_label_en-US" xlink:label="lab_us-gaap_GainLossOnSaleOfProperties" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Gain (Loss) on Sale of Properties</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_GainLossOnSaleOfProperties" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_GainLossOnSaleOfProperties"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_GainLossOnSaleOfProperties" xlink:to="lab_us-gaap_GainLossOnSaleOfProperties" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RealEstateHeldforsale_1fee8018-352f-4c44-8a94-e52c7783ba06_periodStartLabel_en-US" xlink:label="lab_us-gaap_RealEstateHeldforsale" xlink:role="http://www.xbrl.org/2003/role/periodStartLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Beginning Balance</link:label>
    <link:label id="lab_us-gaap_RealEstateHeldforsale_ab65012d-6ca4-40a7-b08f-7c3a59ab69ef_periodEndLabel_en-US" xlink:label="lab_us-gaap_RealEstateHeldforsale" xlink:role="http://www.xbrl.org/2003/role/periodEndLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Ending Balance</link:label>
    <link:label id="lab_us-gaap_RealEstateHeldforsale_label_en-US" xlink:label="lab_us-gaap_RealEstateHeldforsale" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Held-for-sale</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RealEstateHeldforsale" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RealEstateHeldforsale"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RealEstateHeldforsale" xlink:to="lab_us-gaap_RealEstateHeldforsale" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EffectiveIncomeTaxRateContinuingOperations_0deef3d5-9edc-44cd-8b9b-7e447d1075bc_terseLabel_en-US" xlink:label="lab_us-gaap_EffectiveIncomeTaxRateContinuingOperations" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Effective Income Tax Rate Reconciliation, Percent</link:label>
    <link:label id="lab_us-gaap_EffectiveIncomeTaxRateContinuingOperations_label_en-US" xlink:label="lab_us-gaap_EffectiveIncomeTaxRateContinuingOperations" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Effective Income Tax Rate Reconciliation, Percent</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EffectiveIncomeTaxRateContinuingOperations" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EffectiveIncomeTaxRateContinuingOperations"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EffectiveIncomeTaxRateContinuingOperations" xlink:to="lab_us-gaap_EffectiveIncomeTaxRateContinuingOperations" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RealEstateOwnedNumberOfPropertiesRollForward_9d5124b1-c4fc-4bcd-bc3a-f17835a072c9_terseLabel_en-US" xlink:label="lab_efc_RealEstateOwnedNumberOfPropertiesRollForward" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of Properties</link:label>
    <link:label id="lab_efc_RealEstateOwnedNumberOfPropertiesRollForward_label_en-US" xlink:label="lab_efc_RealEstateOwnedNumberOfPropertiesRollForward" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Owned, Number Of Properties [Roll Forward]</link:label>
    <link:label id="lab_efc_RealEstateOwnedNumberOfPropertiesRollForward_documentation_en-US" xlink:label="lab_efc_RealEstateOwnedNumberOfPropertiesRollForward" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Owned, Number Of Properties [Roll Forward]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RealEstateOwnedNumberOfPropertiesRollForward" xlink:href="efc-20201231.xsd#efc_RealEstateOwnedNumberOfPropertiesRollForward"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RealEstateOwnedNumberOfPropertiesRollForward" xlink:to="lab_efc_RealEstateOwnedNumberOfPropertiesRollForward" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleTable_2af2a592-24fd-428f-af2a-0f8c160c486b_terseLabel_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale [Table]</link:label>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleTable_label_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtSecuritiesAvailableForSaleTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtSecuritiesAvailableForSaleTable" xlink:to="lab_us-gaap_DebtSecuritiesAvailableForSaleTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NoncontrollinginterestDomain_ac520294-c08d-4404-a570-0559d08b1138_terseLabel_en-US" xlink:label="lab_efc_NoncontrollinginterestDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling interest [Domain]</link:label>
    <link:label id="lab_efc_NoncontrollinginterestDomain_label_en-US" xlink:label="lab_efc_NoncontrollinginterestDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling interest [Domain]</link:label>
    <link:label id="lab_efc_NoncontrollinginterestDomain_documentation_en-US" xlink:label="lab_efc_NoncontrollinginterestDomain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Domain] for Non-controlling interest [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NoncontrollinginterestDomain" xlink:href="efc-20201231.xsd#efc_NoncontrollinginterestDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NoncontrollinginterestDomain" xlink:to="lab_efc_NoncontrollinginterestDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LongTermDebtFairValue_7bdcc07c-b48c-4e26-94f0-02fea164512b_terseLabel_en-US" xlink:label="lab_us-gaap_LongTermDebtFairValue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Fair Value</link:label>
    <link:label id="lab_us-gaap_LongTermDebtFairValue_label_en-US" xlink:label="lab_us-gaap_LongTermDebtFairValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Fair Value</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LongTermDebtFairValue" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LongTermDebtFairValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LongTermDebtFairValue" xlink:to="lab_us-gaap_LongTermDebtFairValue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NotesissuedbynonQMsecuritizationtrust_cfa59297-3eb8-4b40-80b2-c31cd891ea2d_terseLabel_en-US" xlink:label="lab_efc_NotesissuedbynonQMsecuritizationtrust" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Face Amount of Certificates Issued</link:label>
    <link:label id="lab_efc_NotesissuedbynonQMsecuritizationtrust_label_en-US" xlink:label="lab_efc_NotesissuedbynonQMsecuritizationtrust" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Notes issued by non-QM securitization trust</link:label>
    <link:label id="lab_efc_NotesissuedbynonQMsecuritizationtrust_documentation_en-US" xlink:label="lab_efc_NotesissuedbynonQMsecuritizationtrust" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Original face amount of notes issued by non-QM securitization trust pursuant to a securitization.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NotesissuedbynonQMsecuritizationtrust" xlink:href="efc-20201231.xsd#efc_NotesissuedbynonQMsecuritizationtrust"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NotesissuedbynonQMsecuritizationtrust" xlink:to="lab_efc_NotesissuedbynonQMsecuritizationtrust" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CommonSharesIssuedAndOutstandingUponConversionOfAllConvertibleSecurities_07905500-ce05-4c32-bdbf-3d7ee23820da_terseLabel_en-US" xlink:label="lab_efc_CommonSharesIssuedAndOutstandingUponConversionOfAllConvertibleSecurities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common shares issued and outstanding upon conversion of all convertible securities (in shares)</link:label>
    <link:label id="lab_efc_CommonSharesIssuedAndOutstandingUponConversionOfAllConvertibleSecurities_label_en-US" xlink:label="lab_efc_CommonSharesIssuedAndOutstandingUponConversionOfAllConvertibleSecurities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common Shares Issued And Outstanding Upon Conversion of All Convertible Securities</link:label>
    <link:label id="lab_efc_CommonSharesIssuedAndOutstandingUponConversionOfAllConvertibleSecurities_documentation_en-US" xlink:label="lab_efc_CommonSharesIssuedAndOutstandingUponConversionOfAllConvertibleSecurities" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The Company's common shares issued and outstanding if all of the long term incentive plan units and operating partnership units (owned by the non-controlling interest) that have previously been issued were to be fully vested and exchanged for common shares as of the reporting date.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CommonSharesIssuedAndOutstandingUponConversionOfAllConvertibleSecurities" xlink:href="efc-20201231.xsd#efc_CommonSharesIssuedAndOutstandingUponConversionOfAllConvertibleSecurities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CommonSharesIssuedAndOutstandingUponConversionOfAllConvertibleSecurities" xlink:to="lab_efc_CommonSharesIssuedAndOutstandingUponConversionOfAllConvertibleSecurities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetAccretedDiscountsAmortizedPremiums_ff0f33ef-01d7-4f62-8add-39e43d7d4170_terseLabel_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetAccretedDiscountsAmortizedPremiums" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets, Accreted Discounts/Amortized Premiums</link:label>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetAccretedDiscountsAmortizedPremiums_label_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetAccretedDiscountsAmortizedPremiums" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement With Unobservable Inputs Reconciliation, Recurring Basis, Asset, Accreted Discounts/Amortized Premiums</link:label>
    <link:label id="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetAccretedDiscountsAmortizedPremiums_documentation_en-US" xlink:label="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetAccretedDiscountsAmortizedPremiums" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">The amount of accreted discounts and amortization of premiums related to investments in debt securities classified as Level 3 securities within the fair value heirarchy.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetAccretedDiscountsAmortizedPremiums" xlink:href="efc-20201231.xsd#efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetAccretedDiscountsAmortizedPremiums"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetAccretedDiscountsAmortizedPremiums" xlink:to="lab_efc_FairValueMeasurementWithUnobservableInputsReconciliationRecurringBasisAssetAccretedDiscountsAmortizedPremiums" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentTypeCategorizationMember_0b1a217c-34f1-4335-8424-da8745eb5ac5_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentTypeCategorizationMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments [Domain]</link:label>
    <link:label id="lab_us-gaap_InvestmentTypeCategorizationMember_label_en-US" xlink:label="lab_us-gaap_InvestmentTypeCategorizationMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentTypeCategorizationMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentTypeCategorizationMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentTypeCategorizationMember" xlink:to="lab_us-gaap_InvestmentTypeCategorizationMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfComponentsOfIncomeTaxExpenseBenefitTableTextBlock_b89754b3-6a96-4f84-9878-2e6a8a9230af_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfComponentsOfIncomeTaxExpenseBenefitTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Components of Income Tax Expense (Benefit)</link:label>
    <link:label id="lab_us-gaap_ScheduleOfComponentsOfIncomeTaxExpenseBenefitTableTextBlock_label_en-US" xlink:label="lab_us-gaap_ScheduleOfComponentsOfIncomeTaxExpenseBenefitTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Components of Income Tax Expense (Benefit) [Table Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfComponentsOfIncomeTaxExpenseBenefitTableTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfComponentsOfIncomeTaxExpenseBenefitTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfComponentsOfIncomeTaxExpenseBenefitTableTextBlock" xlink:to="lab_us-gaap_ScheduleOfComponentsOfIncomeTaxExpenseBenefitTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_Liabilities_691d2c2b-7ec5-445b-9669-582999a0db6d_totalLabel_en-US" xlink:label="lab_us-gaap_Liabilities" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Liabilities</link:label>
    <link:label id="lab_us-gaap_Liabilities_label_en-US" xlink:label="lab_us-gaap_Liabilities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_Liabilities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_Liabilities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_Liabilities" xlink:to="lab_us-gaap_Liabilities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Outstandingdebtofrelatedparty_76a5f794-ea50-4a63-beae-0566a0cfa67a_terseLabel_en-US" xlink:label="lab_efc_Outstandingdebtofrelatedparty" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Outstanding debt of related party</link:label>
    <link:label id="lab_efc_Outstandingdebtofrelatedparty_label_en-US" xlink:label="lab_efc_Outstandingdebtofrelatedparty" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Outstanding debt of related party</link:label>
    <link:label id="lab_efc_Outstandingdebtofrelatedparty_documentation_en-US" xlink:label="lab_efc_Outstandingdebtofrelatedparty" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total outstanding debt of related party entity for which the Company co owns along with various other related parties. Outstanding debt is in the form of a reverse repurchase agreement (sale of assets under an agreement to repurchase).</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Outstandingdebtofrelatedparty" xlink:href="efc-20201231.xsd#efc_Outstandingdebtofrelatedparty"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Outstandingdebtofrelatedparty" xlink:to="lab_efc_Outstandingdebtofrelatedparty" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CommitmentToPurchaseLoans_9c60856e-5258-43d7-8d27-6bf9bd5a7e6b_terseLabel_en-US" xlink:label="lab_efc_CommitmentToPurchaseLoans" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commitment to purchase loans</link:label>
    <link:label id="lab_efc_CommitmentToPurchaseLoans_label_en-US" xlink:label="lab_efc_CommitmentToPurchaseLoans" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commitment to purchase loans</link:label>
    <link:label id="lab_efc_CommitmentToPurchaseLoans_documentation_en-US" xlink:label="lab_efc_CommitmentToPurchaseLoans" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Commitment to purchase loans</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CommitmentToPurchaseLoans" xlink:href="efc-20201231.xsd#efc_CommitmentToPurchaseLoans"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CommitmentToPurchaseLoans" xlink:to="lab_efc_CommitmentToPurchaseLoans" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RepaymentsofReverseRepurchaseAgreements_535de04a-b831-4f35-90e2-c74c19e2bd9e_negatedLabel_en-US" xlink:label="lab_efc_RepaymentsofReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repayments of repurchase agreements</link:label>
    <link:label id="lab_efc_RepaymentsofReverseRepurchaseAgreements_label_en-US" xlink:label="lab_efc_RepaymentsofReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repayments of Reverse Repurchase Agreements</link:label>
    <link:label id="lab_efc_RepaymentsofReverseRepurchaseAgreements_documentation_en-US" xlink:label="lab_efc_RepaymentsofReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash outflow from repayments of borrowings from securities sold under agreements to repurchase.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RepaymentsofReverseRepurchaseAgreements" xlink:href="efc-20201231.xsd#efc_RepaymentsofReverseRepurchaseAgreements"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RepaymentsofReverseRepurchaseAgreements" xlink:to="lab_efc_RepaymentsofReverseRepurchaseAgreements" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfAvailableForSaleSecurities_7fe3931e-08d1-4dd5-9147-9be86707dd76_terseLabel_en-US" xlink:label="lab_us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfAvailableForSaleSecurities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from principal payments of securities</link:label>
    <link:label id="lab_us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfAvailableForSaleSecurities_label_en-US" xlink:label="lab_us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfAvailableForSaleSecurities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from Maturities, Prepayments and Calls of Debt Securities, Available-for-sale</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfAvailableForSaleSecurities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfAvailableForSaleSecurities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfAvailableForSaleSecurities" xlink:to="lab_us-gaap_ProceedsFromMaturitiesPrepaymentsAndCallsOfAvailableForSaleSecurities" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_November2019Member_cb931b49-3bf9-4fcb-9c59-f5f832054aa7_terseLabel_en-US" xlink:label="lab_efc_November2019Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">November 2019 [Member]</link:label>
    <link:label id="lab_efc_November2019Member_label_en-US" xlink:label="lab_efc_November2019Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">November 2019 [Member]</link:label>
    <link:label id="lab_efc_November2019Member_documentation_en-US" xlink:label="lab_efc_November2019Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">November 2019 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_November2019Member" xlink:href="efc-20201231.xsd#efc_November2019Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_November2019Member" xlink:to="lab_efc_November2019Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RealEstateOwnedAcquiredThroughForeclosureFairValue_8c27a8a1-15be-4d0d-aee9-ace5e6483168_terseLabel_en-US" xlink:label="lab_efc_RealEstateOwnedAcquiredThroughForeclosureFairValue" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">REO, fair value</link:label>
    <link:label id="lab_efc_RealEstateOwnedAcquiredThroughForeclosureFairValue_label_en-US" xlink:label="lab_efc_RealEstateOwnedAcquiredThroughForeclosureFairValue" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Owned, Acquired Through Foreclosure, Fair Value</link:label>
    <link:label id="lab_efc_RealEstateOwnedAcquiredThroughForeclosureFairValue_documentation_en-US" xlink:label="lab_efc_RealEstateOwnedAcquiredThroughForeclosureFairValue" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount of real estate owned held at fair value at the reporting date. Real estate owned properties are held at the lower of cost or fair value.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RealEstateOwnedAcquiredThroughForeclosureFairValue" xlink:href="efc-20201231.xsd#efc_RealEstateOwnedAcquiredThroughForeclosureFairValue"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RealEstateOwnedAcquiredThroughForeclosureFairValue" xlink:to="lab_efc_RealEstateOwnedAcquiredThroughForeclosureFairValue" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_OtherStatesMember_48792658-96cc-4b35-b67a-355ad6cd534b_terseLabel_en-US" xlink:label="lab_efc_OtherStatesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other [Member]</link:label>
    <link:label id="lab_efc_OtherStatesMember_label_en-US" xlink:label="lab_efc_OtherStatesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other States [Member]</link:label>
    <link:label id="lab_efc_OtherStatesMember_documentation_en-US" xlink:label="lab_efc_OtherStatesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other States [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_OtherStatesMember" xlink:href="efc-20201231.xsd#efc_OtherStatesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_OtherStatesMember" xlink:to="lab_efc_OtherStatesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SingleFamilyHousesMember_7b20a088-e532-4347-8202-88bb6fc48fc6_terseLabel_en-US" xlink:label="lab_efc_SingleFamilyHousesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Single-Family Houses [Member]</link:label>
    <link:label id="lab_efc_SingleFamilyHousesMember_label_en-US" xlink:label="lab_efc_SingleFamilyHousesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Single-Family Houses [Member]</link:label>
    <link:label id="lab_efc_SingleFamilyHousesMember_documentation_en-US" xlink:label="lab_efc_SingleFamilyHousesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Single-Family Houses [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SingleFamilyHousesMember" xlink:href="efc-20201231.xsd#efc_SingleFamilyHousesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SingleFamilyHousesMember" xlink:to="lab_efc_SingleFamilyHousesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2036Member_817b651f-85ed-447e-a856-f7c3a079e282_terseLabel_en-US" xlink:label="lab_efc_A2036Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2036 [Member]</link:label>
    <link:label id="lab_efc_A2036Member_label_en-US" xlink:label="lab_efc_A2036Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2036 [Member]</link:label>
    <link:label id="lab_efc_A2036Member_documentation_en-US" xlink:label="lab_efc_A2036Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2036 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2036Member" xlink:href="efc-20201231.xsd#efc_A2036Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2036Member" xlink:to="lab_efc_A2036Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FairValueOfLevel3AssetsNegativeYield_8ec8c445-3f94-4983-8698-62b3b971cc30_terseLabel_en-US" xlink:label="lab_efc_FairValueOfLevel3AssetsNegativeYield" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value of Level 3 Assets, Negative Yield</link:label>
    <link:label id="lab_efc_FairValueOfLevel3AssetsNegativeYield_label_en-US" xlink:label="lab_efc_FairValueOfLevel3AssetsNegativeYield" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value of Level 3 Assets, Negative Yield</link:label>
    <link:label id="lab_efc_FairValueOfLevel3AssetsNegativeYield_documentation_en-US" xlink:label="lab_efc_FairValueOfLevel3AssetsNegativeYield" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value of Level 3 Assets, Negative Yield</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FairValueOfLevel3AssetsNegativeYield" xlink:href="efc-20201231.xsd#efc_FairValueOfLevel3AssetsNegativeYield"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FairValueOfLevel3AssetsNegativeYield" xlink:to="lab_efc_FairValueOfLevel3AssetsNegativeYield" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_RepurchaseAgreementCounterpartyNameDomain_3db7de7d-aa62-45cd-81ab-7bced6f18a10_terseLabel_en-US" xlink:label="lab_srt_RepurchaseAgreementCounterpartyNameDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Counterparty Name [Domain]</link:label>
    <link:label id="lab_srt_RepurchaseAgreementCounterpartyNameDomain_label_en-US" xlink:label="lab_srt_RepurchaseAgreementCounterpartyNameDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Counterparty Name [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_RepurchaseAgreementCounterpartyNameDomain" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_RepurchaseAgreementCounterpartyNameDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_RepurchaseAgreementCounterpartyNameDomain" xlink:to="lab_srt_RepurchaseAgreementCounterpartyNameDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancingReceivablesPeriodPastDueAxis_418901e9-ef55-46d9-aec5-caf557b36933_terseLabel_en-US" xlink:label="lab_us-gaap_FinancingReceivablesPeriodPastDueAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Asset, Period Past Due [Axis]</link:label>
    <link:label id="lab_us-gaap_FinancingReceivablesPeriodPastDueAxis_label_en-US" xlink:label="lab_us-gaap_FinancingReceivablesPeriodPastDueAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Asset, Period Past Due [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancingReceivablesPeriodPastDueAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancingReceivablesPeriodPastDueAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancingReceivablesPeriodPastDueAxis" xlink:to="lab_us-gaap_FinancingReceivablesPeriodPastDueAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleAllowanceForCreditLoss_2f57581d-ec8b-4559-b93f-1d33be3dfc49_terseLabel_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleAllowanceForCreditLoss" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Allowance for Credit Loss</link:label>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleAllowanceForCreditLoss_label_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleAllowanceForCreditLoss" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Allowance for Credit Loss</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtSecuritiesAvailableForSaleAllowanceForCreditLoss" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleAllowanceForCreditLoss"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtSecuritiesAvailableForSaleAllowanceForCreditLoss" xlink:to="lab_us-gaap_DebtSecuritiesAvailableForSaleAllowanceForCreditLoss" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ProceedsFromDispositionofRealEstateOwned_25353b92-e2ec-46c2-b258-c2229abb2000_terseLabel_en-US" xlink:label="lab_efc_ProceedsFromDispositionofRealEstateOwned" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from disposition of real estate owned</link:label>
    <link:label id="lab_efc_ProceedsFromDispositionofRealEstateOwned_label_en-US" xlink:label="lab_efc_ProceedsFromDispositionofRealEstateOwned" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds From Disposition of Real Estate Owned</link:label>
    <link:label id="lab_efc_ProceedsFromDispositionofRealEstateOwned_documentation_en-US" xlink:label="lab_efc_ProceedsFromDispositionofRealEstateOwned" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds From Disposition of Real Estate Owned</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ProceedsFromDispositionofRealEstateOwned" xlink:href="efc-20201231.xsd#efc_ProceedsFromDispositionofRealEstateOwned"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ProceedsFromDispositionofRealEstateOwned" xlink:to="lab_efc_ProceedsFromDispositionofRealEstateOwned" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NonvotingCommonStockMember_5f4fd2d3-a162-4dd3-adad-b29b06e6d58f_terseLabel_en-US" xlink:label="lab_us-gaap_NonvotingCommonStockMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Nonvoting Common Stock [Member]</link:label>
    <link:label id="lab_us-gaap_NonvotingCommonStockMember_label_en-US" xlink:label="lab_us-gaap_NonvotingCommonStockMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Nonvoting Common Stock [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NonvotingCommonStockMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NonvotingCommonStockMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NonvotingCommonStockMember" xlink:to="lab_us-gaap_NonvotingCommonStockMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_J.P.MorganSecuritiesLLCMember_18981b41-b154-4c09-9ea7-367a494865f0_terseLabel_en-US" xlink:label="lab_efc_J.P.MorganSecuritiesLLCMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">J.P. Morgan Securities LLC [Member]</link:label>
    <link:label id="lab_efc_J.P.MorganSecuritiesLLCMember_label_en-US" xlink:label="lab_efc_J.P.MorganSecuritiesLLCMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">J.P. Morgan Securities LLC [Member]</link:label>
    <link:label id="lab_efc_J.P.MorganSecuritiesLLCMember_documentation_en-US" xlink:label="lab_efc_J.P.MorganSecuritiesLLCMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">J.P. Morgan Securities LLC [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_J.P.MorganSecuritiesLLCMember" xlink:href="efc-20201231.xsd#efc_J.P.MorganSecuritiesLLCMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_J.P.MorganSecuritiesLLCMember" xlink:to="lab_efc_J.P.MorganSecuritiesLLCMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ForeignCurrencyTransactionGainLossUnrealized_5dd045bb-6081-485d-91f0-e62731f1940a_terseLabel_en-US" xlink:label="lab_us-gaap_ForeignCurrencyTransactionGainLossUnrealized" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Foreign currency translation</link:label>
    <link:label id="lab_us-gaap_ForeignCurrencyTransactionGainLossUnrealized_label_en-US" xlink:label="lab_us-gaap_ForeignCurrencyTransactionGainLossUnrealized" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Foreign Currency Transaction Gain (Loss), Unrealized</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ForeignCurrencyTransactionGainLossUnrealized" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ForeignCurrencyTransactionGainLossUnrealized"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ForeignCurrencyTransactionGainLossUnrealized" xlink:to="lab_us-gaap_ForeignCurrencyTransactionGainLossUnrealized" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureDisposalsNumberOfProperties_a8a6eec5-feb4-45a4-aa4a-fb1adb524ca4_terseLabel_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureDisposalsNumberOfProperties" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure, Disposals, Number Of Properties</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureDisposalsNumberOfProperties_76a42623-49ce-4168-aa69-ee3dc2d9e749_negatedLabel_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureDisposalsNumberOfProperties" xlink:role="http://www.xbrl.org/2009/role/negatedLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disposals</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureDisposalsNumberOfProperties_label_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureDisposalsNumberOfProperties" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure, Disposals, Number Of Properties</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureDisposalsNumberOfProperties_documentation_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureDisposalsNumberOfProperties" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure, Disposals, Number Of Properties</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RealEstateAcquiredThroughForeclosureDisposalsNumberOfProperties" xlink:href="efc-20201231.xsd#efc_RealEstateAcquiredThroughForeclosureDisposalsNumberOfProperties"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RealEstateAcquiredThroughForeclosureDisposalsNumberOfProperties" xlink:to="lab_efc_RealEstateAcquiredThroughForeclosureDisposalsNumberOfProperties" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InvestmentssoldnotyetpurchasedPolicyPolicyTextBlock_ad1c9017-d3fa-44e1-9a5a-1814ccd05742_terseLabel_en-US" xlink:label="lab_efc_InvestmentssoldnotyetpurchasedPolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities Sold Short</link:label>
    <link:label id="lab_efc_InvestmentssoldnotyetpurchasedPolicyPolicyTextBlock_label_en-US" xlink:label="lab_efc_InvestmentssoldnotyetpurchasedPolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments sold, not yet purchased Policy [Policy Text Block]</link:label>
    <link:label id="lab_efc_InvestmentssoldnotyetpurchasedPolicyPolicyTextBlock_documentation_en-US" xlink:label="lab_efc_InvestmentssoldnotyetpurchasedPolicyPolicyTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Disclosure of accounting policy for securities sold, not yet purchased (securities sold short).</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InvestmentssoldnotyetpurchasedPolicyPolicyTextBlock" xlink:href="efc-20201231.xsd#efc_InvestmentssoldnotyetpurchasedPolicyPolicyTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InvestmentssoldnotyetpurchasedPolicyPolicyTextBlock" xlink:to="lab_efc_InvestmentssoldnotyetpurchasedPolicyPolicyTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NetRealizedGainLossonFinancialDerivativesexcludingcurrencyhedges_13819910-b0b2-4a36-8c22-a7e08cec56fd_terseLabel_en-US" xlink:label="lab_efc_NetRealizedGainLossonFinancialDerivativesexcludingcurrencyhedges" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial derivatives, excluding currency hedges</link:label>
    <link:label id="lab_efc_NetRealizedGainLossonFinancialDerivativesexcludingcurrencyhedges_label_en-US" xlink:label="lab_efc_NetRealizedGainLossonFinancialDerivativesexcludingcurrencyhedges" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Realized Gain/Loss on Financial Derivatives, excluding currency hedges</link:label>
    <link:label id="lab_efc_NetRealizedGainLossonFinancialDerivativesexcludingcurrencyhedges_documentation_en-US" xlink:label="lab_efc_NetRealizedGainLossonFinancialDerivativesexcludingcurrencyhedges" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Realized Gain/Loss on Financial Derivatives, excluding currency hedges</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NetRealizedGainLossonFinancialDerivativesexcludingcurrencyhedges" xlink:href="efc-20201231.xsd#efc_NetRealizedGainLossonFinancialDerivativesexcludingcurrencyhedges"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NetRealizedGainLossonFinancialDerivativesexcludingcurrencyhedges" xlink:to="lab_efc_NetRealizedGainLossonFinancialDerivativesexcludingcurrencyhedges" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CorporatedebtnonexchangetradedcorporateequityandsecurednotesMember_10a626a9-a0e9-40fc-b93c-242d940c7957_terseLabel_en-US" xlink:label="lab_efc_CorporatedebtnonexchangetradedcorporateequityandsecurednotesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate debt, non-exchange traded corporate equity, and secured notes [Member]</link:label>
    <link:label id="lab_efc_CorporatedebtnonexchangetradedcorporateequityandsecurednotesMember_label_en-US" xlink:label="lab_efc_CorporatedebtnonexchangetradedcorporateequityandsecurednotesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate debt, non-exchange traded corporate equity, and secured notes [Member]</link:label>
    <link:label id="lab_efc_CorporatedebtnonexchangetradedcorporateequityandsecurednotesMember_documentation_en-US" xlink:label="lab_efc_CorporatedebtnonexchangetradedcorporateequityandsecurednotesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Corporate debt, non-exchange traded corporate equity, and secured notes [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CorporatedebtnonexchangetradedcorporateequityandsecurednotesMember" xlink:href="efc-20201231.xsd#efc_CorporatedebtnonexchangetradedcorporateequityandsecurednotesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CorporatedebtnonexchangetradedcorporateequityandsecurednotesMember" xlink:to="lab_efc_CorporatedebtnonexchangetradedcorporateequityandsecurednotesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_JepsonHoldingsLimitedMember_979804c4-fa22-4bec-b2b5-b6c5c4f3a883_terseLabel_en-US" xlink:label="lab_efc_JepsonHoldingsLimitedMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Jepson Holdings Limited</link:label>
    <link:label id="lab_efc_JepsonHoldingsLimitedMember_label_en-US" xlink:label="lab_efc_JepsonHoldingsLimitedMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Jepson Holdings Limited [Member]</link:label>
    <link:label id="lab_efc_JepsonHoldingsLimitedMember_documentation_en-US" xlink:label="lab_efc_JepsonHoldingsLimitedMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Jepson Holdings Limited [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_JepsonHoldingsLimitedMember" xlink:href="efc-20201231.xsd#efc_JepsonHoldingsLimitedMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_JepsonHoldingsLimitedMember" xlink:to="lab_efc_JepsonHoldingsLimitedMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentsAbstract_e6290f0d-e253-4ed9-8613-cf3a6581868e_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments, financial derivatives, and repurchase agreements:</link:label>
    <link:label id="lab_us-gaap_InvestmentsAbstract_label_en-US" xlink:label="lab_us-gaap_InvestmentsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investments [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentsAbstract" xlink:to="lab_us-gaap_InvestmentsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Transfersfrommortgageloanstoinvestmentsinnonconsolidatedentitiesnoncash_ce790390-f12a-437c-a9a6-7afdd1d7cb16_terseLabel_en-US" xlink:label="lab_efc_Transfersfrommortgageloanstoinvestmentsinnonconsolidatedentitiesnoncash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transfers from mortgage loans to investments in non-consolidated entities (non-cash)</link:label>
    <link:label id="lab_efc_Transfersfrommortgageloanstoinvestmentsinnonconsolidatedentitiesnoncash_label_en-US" xlink:label="lab_efc_Transfersfrommortgageloanstoinvestmentsinnonconsolidatedentitiesnoncash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transfers from mortgage loans to investments in non-consolidated entities (non-cash)</link:label>
    <link:label id="lab_efc_Transfersfrommortgageloanstoinvestmentsinnonconsolidatedentitiesnoncash_documentation_en-US" xlink:label="lab_efc_Transfersfrommortgageloanstoinvestmentsinnonconsolidatedentitiesnoncash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Transfers from mortgage loans to investments in non-consolidated entities (non-cash)</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Transfersfrommortgageloanstoinvestmentsinnonconsolidatedentitiesnoncash" xlink:href="efc-20201231.xsd#efc_Transfersfrommortgageloanstoinvestmentsinnonconsolidatedentitiesnoncash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Transfersfrommortgageloanstoinvestmentsinnonconsolidatedentitiesnoncash" xlink:to="lab_efc_Transfersfrommortgageloanstoinvestmentsinnonconsolidatedentitiesnoncash" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleMeasurementInput_6dfda7a0-5988-4cc5-99dc-187d4ac79506_terseLabel_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securities, measurement input</link:label>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleMeasurementInput_label_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Measurement Input</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtSecuritiesAvailableForSaleMeasurementInput" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleMeasurementInput"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtSecuritiesAvailableForSaleMeasurementInput" xlink:to="lab_us-gaap_DebtSecuritiesAvailableForSaleMeasurementInput" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsMember_c21a5a74-e33b-44f8-8fcb-7ce8f5a3955b_terseLabel_en-US" xlink:label="lab_us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Restricted Cash [Domain]</link:label>
    <link:label id="lab_us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsMember_label_en-US" xlink:label="lab_us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash and Cash Equivalents [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsMember" xlink:to="lab_us-gaap_RestrictedCashAndCashEquivalentsCashAndCashEquivalentsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeMember_bd17a549-5493-4444-aa9e-1764c694963e_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative [Member]</link:label>
    <link:label id="lab_us-gaap_DerivativeMember_label_en-US" xlink:label="lab_us-gaap_DerivativeMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeMember" xlink:to="lab_us-gaap_DerivativeMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NoncontrollinginterestofjointventurepartnerMember_3c439bad-660c-43a9-af2a-01cc3168b760_terseLabel_en-US" xlink:label="lab_efc_NoncontrollinginterestofjointventurepartnerMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling interest of joint venture partner</link:label>
    <link:label id="lab_efc_NoncontrollinginterestofjointventurepartnerMember_label_en-US" xlink:label="lab_efc_NoncontrollinginterestofjointventurepartnerMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling interest of joint venture partner [Member]</link:label>
    <link:label id="lab_efc_NoncontrollinginterestofjointventurepartnerMember_documentation_en-US" xlink:label="lab_efc_NoncontrollinginterestofjointventurepartnerMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Non-controlling interest of joint venture partner in the Company. Interest is not convertible into common shares of the Company.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NoncontrollinginterestofjointventurepartnerMember" xlink:href="efc-20201231.xsd#efc_NoncontrollinginterestofjointventurepartnerMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NoncontrollinginterestofjointventurepartnerMember" xlink:to="lab_efc_NoncontrollinginterestofjointventurepartnerMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_VariableInterestEntityEntityMaximumLossExposureAmount_883dcb3e-b33d-4329-8858-9c3e15716d76_terseLabel_en-US" xlink:label="lab_us-gaap_VariableInterestEntityEntityMaximumLossExposureAmount" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Variable Interest Entity, Maximum Loss Exposure</link:label>
    <link:label id="lab_us-gaap_VariableInterestEntityEntityMaximumLossExposureAmount_label_en-US" xlink:label="lab_us-gaap_VariableInterestEntityEntityMaximumLossExposureAmount" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Variable Interest Entity, Reporting Entity Involvement, Maximum Loss Exposure, Amount</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_VariableInterestEntityEntityMaximumLossExposureAmount" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_VariableInterestEntityEntityMaximumLossExposureAmount"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_VariableInterestEntityEntityMaximumLossExposureAmount" xlink:to="lab_us-gaap_VariableInterestEntityEntityMaximumLossExposureAmount" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StockholdersEquity_a854b40a-202f-44c4-833e-a4eda2a5f5aa_totalLabel_en-US" xlink:label="lab_us-gaap_StockholdersEquity" xlink:role="http://www.xbrl.org/2003/role/totalLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Total Stockholders' Equity</link:label>
    <link:label id="lab_us-gaap_StockholdersEquity_label_en-US" xlink:label="lab_us-gaap_StockholdersEquity" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stockholders' Equity Attributable to Parent</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StockholdersEquity" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockholdersEquity"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StockholdersEquity" xlink:to="lab_us-gaap_StockholdersEquity" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ConcentrationRiskTypeDomain_823cbf37-89a6-49f3-a87b-2f4d9bd80cc3_terseLabel_en-US" xlink:label="lab_us-gaap_ConcentrationRiskTypeDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk Type [Domain]</link:label>
    <link:label id="lab_us-gaap_ConcentrationRiskTypeDomain_label_en-US" xlink:label="lab_us-gaap_ConcentrationRiskTypeDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk Type [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ConcentrationRiskTypeDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ConcentrationRiskTypeDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ConcentrationRiskTypeDomain" xlink:to="lab_us-gaap_ConcentrationRiskTypeDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_IncreaseDecreaseInOtherOperatingLiabilities_383bb02c-4dca-4698-8117-ccdc177fbdb6_terseLabel_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInOtherOperatingLiabilities" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other liabilities</link:label>
    <link:label id="lab_us-gaap_IncreaseDecreaseInOtherOperatingLiabilities_label_en-US" xlink:label="lab_us-gaap_IncreaseDecreaseInOtherOperatingLiabilities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Increase (Decrease) in Other Operating Liabilities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_IncreaseDecreaseInOtherOperatingLiabilities" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncreaseDecreaseInOtherOperatingLiabilities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_IncreaseDecreaseInOtherOperatingLiabilities" xlink:to="lab_us-gaap_IncreaseDecreaseInOtherOperatingLiabilities" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancingReceivableAllowanceForCreditLossPurchasedWithCreditDeteriorationIncrease_0fc6a460-c04d-43c2-9e3c-afdff3b07ded_terseLabel_en-US" xlink:label="lab_us-gaap_FinancingReceivableAllowanceForCreditLossPurchasedWithCreditDeteriorationIncrease" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable, Allowance for Credit Loss</link:label>
    <link:label id="lab_us-gaap_FinancingReceivableAllowanceForCreditLossPurchasedWithCreditDeteriorationIncrease_label_en-US" xlink:label="lab_us-gaap_FinancingReceivableAllowanceForCreditLossPurchasedWithCreditDeteriorationIncrease" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financing Receivable, Allowance for Credit Loss, Purchased with Credit Deterioration, Increase</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancingReceivableAllowanceForCreditLossPurchasedWithCreditDeteriorationIncrease" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancingReceivableAllowanceForCreditLossPurchasedWithCreditDeteriorationIncrease"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancingReceivableAllowanceForCreditLossPurchasedWithCreditDeteriorationIncrease" xlink:to="lab_us-gaap_FinancingReceivableAllowanceForCreditLossPurchasedWithCreditDeteriorationIncrease" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySalesAndIssues_14c4fc4d-1dbd-492f-a594-e154ede85ebb_negatedTerseLabel_en-US" xlink:label="lab_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySalesAndIssues" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities, Sales/Issuance</link:label>
    <link:label id="lab_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySalesAndIssues_label_en-US" xlink:label="lab_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySalesAndIssues" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurment with Unobservable Inputs Reconciliation, Recurring Basis, Liability, Sales and Issues</link:label>
    <link:label id="lab_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySalesAndIssues_documentation_en-US" xlink:label="lab_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySalesAndIssues" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Sales and issues that have taken place during the period in relation to liabilities measured at fair value and categorized within Level 3 of the fair value hierarchy.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySalesAndIssues" xlink:href="efc-20201231.xsd#efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySalesAndIssues"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySalesAndIssues" xlink:to="lab_efc_FairValueMeasurmentWithUnobservableInputsReconciliationRecurringBasisLiabilitySalesAndIssues" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CashDividendsTaxCharacteristicTableTextBlock_4496c0b1-7102-462a-bf77-06c3bb1d666f_terseLabel_en-US" xlink:label="lab_efc_CashDividendsTaxCharacteristicTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash Dividends, Tax Characteristic</link:label>
    <link:label id="lab_efc_CashDividendsTaxCharacteristicTableTextBlock_label_en-US" xlink:label="lab_efc_CashDividendsTaxCharacteristicTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash Dividends, Tax Characteristic [Table Text Block]</link:label>
    <link:label id="lab_efc_CashDividendsTaxCharacteristicTableTextBlock_documentation_en-US" xlink:label="lab_efc_CashDividendsTaxCharacteristicTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash Dividends, Tax Characteristic</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CashDividendsTaxCharacteristicTableTextBlock" xlink:href="efc-20201231.xsd#efc_CashDividendsTaxCharacteristicTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CashDividendsTaxCharacteristicTableTextBlock" xlink:to="lab_efc_CashDividendsTaxCharacteristicTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_LoansMeasurementInput_bc1ec66b-9e4c-4191-94a1-6e0ff50a7725_terseLabel_en-US" xlink:label="lab_efc_LoansMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans, Measurement Input</link:label>
    <link:label id="lab_efc_LoansMeasurementInput_label_en-US" xlink:label="lab_efc_LoansMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Loans, Measurement Input</link:label>
    <link:label id="lab_efc_LoansMeasurementInput_documentation_en-US" xlink:label="lab_efc_LoansMeasurementInput" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Value of input used to measure loan.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_LoansMeasurementInput" xlink:href="efc-20201231.xsd#efc_LoansMeasurementInput"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_LoansMeasurementInput" xlink:to="lab_efc_LoansMeasurementInput" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Sharebasedlongtermincentiveplanunitredemptionnoncash_b8a2c365-b5b0-4ca8-97ae-874e49c0fceb_terseLabel_en-US" xlink:label="lab_efc_Sharebasedlongtermincentiveplanunitredemptionnoncash" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share based long term incentive plan unit redemption non cash</link:label>
    <link:label id="lab_efc_Sharebasedlongtermincentiveplanunitredemptionnoncash_label_en-US" xlink:label="lab_efc_Sharebasedlongtermincentiveplanunitredemptionnoncash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share based long term incentive plan unit redemption non cash</link:label>
    <link:label id="lab_efc_Sharebasedlongtermincentiveplanunitredemptionnoncash_documentation_en-US" xlink:label="lab_efc_Sharebasedlongtermincentiveplanunitredemptionnoncash" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Share based long term incentive plan unit redemption non cash</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Sharebasedlongtermincentiveplanunitredemptionnoncash" xlink:href="efc-20201231.xsd#efc_Sharebasedlongtermincentiveplanunitredemptionnoncash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Sharebasedlongtermincentiveplanunitredemptionnoncash" xlink:to="lab_efc_Sharebasedlongtermincentiveplanunitredemptionnoncash" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_UnderlyingAssetClassDomain_b72df183-4af4-4299-b7bd-5d7a7bcddfc6_terseLabel_en-US" xlink:label="lab_us-gaap_UnderlyingAssetClassDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Underlying Asset Class [Domain]</link:label>
    <link:label id="lab_us-gaap_UnderlyingAssetClassDomain_label_en-US" xlink:label="lab_us-gaap_UnderlyingAssetClassDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Underlying Asset Class [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_UnderlyingAssetClassDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_UnderlyingAssetClassDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_UnderlyingAssetClassDomain" xlink:to="lab_us-gaap_UnderlyingAssetClassDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ConcentrationRiskByBenchmarkAxis_f7fbd424-9c81-4347-b5c4-b220daee23cc_terseLabel_en-US" xlink:label="lab_us-gaap_ConcentrationRiskByBenchmarkAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk Benchmark [Axis]</link:label>
    <link:label id="lab_us-gaap_ConcentrationRiskByBenchmarkAxis_label_en-US" xlink:label="lab_us-gaap_ConcentrationRiskByBenchmarkAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk Benchmark [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ConcentrationRiskByBenchmarkAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ConcentrationRiskByBenchmarkAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ConcentrationRiskByBenchmarkAxis" xlink:to="lab_us-gaap_ConcentrationRiskByBenchmarkAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentdebtMember_a7a5c77d-4507-4f19-b39e-b48090562af3_terseLabel_en-US" xlink:label="lab_efc_GovernmentdebtMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government debt</link:label>
    <link:label id="lab_efc_GovernmentdebtMember_label_en-US" xlink:label="lab_efc_GovernmentdebtMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government debt [Member]</link:label>
    <link:label id="lab_efc_GovernmentdebtMember_documentation_en-US" xlink:label="lab_efc_GovernmentdebtMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt issued by a governmental body. Can include debt issued by foreign national governments or the US Government.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentdebtMember" xlink:href="efc-20201231.xsd#efc_GovernmentdebtMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentdebtMember" xlink:to="lab_efc_GovernmentdebtMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FairValueofPurchasedCreditDerivativesOffsettingWrittenCreditDerivatives_9a509b02-14e9-4f3a-a153-0383cd9a2303_negatedTerseLabel_en-US" xlink:label="lab_efc_FairValueofPurchasedCreditDerivativesOffsettingWrittenCreditDerivatives" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value of Purchased Credit Derivatives Offsetting Written Credit Derivatives</link:label>
    <link:label id="lab_efc_FairValueofPurchasedCreditDerivativesOffsettingWrittenCreditDerivatives_label_en-US" xlink:label="lab_efc_FairValueofPurchasedCreditDerivativesOffsettingWrittenCreditDerivatives" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value of Purchased Credit Derivatives Offsetting Written Credit Derivatives</link:label>
    <link:label id="lab_efc_FairValueofPurchasedCreditDerivativesOffsettingWrittenCreditDerivatives_documentation_en-US" xlink:label="lab_efc_FairValueofPurchasedCreditDerivativesOffsettingWrittenCreditDerivatives" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair value of purchased credit derivatives (short or purchased protection) offsetting written credit derivatives (long or sold protection).</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FairValueofPurchasedCreditDerivativesOffsettingWrittenCreditDerivatives" xlink:href="efc-20201231.xsd#efc_FairValueofPurchasedCreditDerivativesOffsettingWrittenCreditDerivatives"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FairValueofPurchasedCreditDerivativesOffsettingWrittenCreditDerivatives" xlink:to="lab_efc_FairValueofPurchasedCreditDerivativesOffsettingWrittenCreditDerivatives" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NoncontrollingInterestIncreaseFromSaleOfParentEquityInterest_30ec1aac-f802-4b21-89a7-4784b8f2b8b3_terseLabel_en-US" xlink:label="lab_us-gaap_NoncontrollingInterestIncreaseFromSaleOfParentEquityInterest" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Contributions from non-controlling interests</link:label>
    <link:label id="lab_us-gaap_NoncontrollingInterestIncreaseFromSaleOfParentEquityInterest_label_en-US" xlink:label="lab_us-gaap_NoncontrollingInterestIncreaseFromSaleOfParentEquityInterest" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Noncontrolling Interest, Increase from Sale of Parent Equity Interest</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NoncontrollingInterestIncreaseFromSaleOfParentEquityInterest" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NoncontrollingInterestIncreaseFromSaleOfParentEquityInterest"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NoncontrollingInterestIncreaseFromSaleOfParentEquityInterest" xlink:to="lab_us-gaap_NoncontrollingInterestIncreaseFromSaleOfParentEquityInterest" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ScheduleOfRelatedPartyTransactionsByRelatedPartyTable_4b07ff6e-876f-4960-b12c-abb45311b247_terseLabel_en-US" xlink:label="lab_us-gaap_ScheduleOfRelatedPartyTransactionsByRelatedPartyTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Related Party Transactions, by Related Party [Table]</link:label>
    <link:label id="lab_us-gaap_ScheduleOfRelatedPartyTransactionsByRelatedPartyTable_label_en-US" xlink:label="lab_us-gaap_ScheduleOfRelatedPartyTransactionsByRelatedPartyTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Related Party Transactions, by Related Party [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ScheduleOfRelatedPartyTransactionsByRelatedPartyTable" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ScheduleOfRelatedPartyTransactionsByRelatedPartyTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ScheduleOfRelatedPartyTransactionsByRelatedPartyTable" xlink:to="lab_us-gaap_ScheduleOfRelatedPartyTransactionsByRelatedPartyTable" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueDisclosuresTextBlock_6226454d-76de-494c-9c4b-a56f9e66cf04_verboseLabel_en-US" xlink:label="lab_us-gaap_FairValueDisclosuresTextBlock" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Valuation</link:label>
    <link:label id="lab_us-gaap_FairValueDisclosuresTextBlock_label_en-US" xlink:label="lab_us-gaap_FairValueDisclosuresTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value Disclosures [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueDisclosuresTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueDisclosuresTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueDisclosuresTextBlock" xlink:to="lab_us-gaap_FairValueDisclosuresTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_OtherExpenses_c75726e2-4fd4-4f56-8a69-ed67dd22fe71_terseLabel_en-US" xlink:label="lab_us-gaap_OtherExpenses" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other expenses</link:label>
    <link:label id="lab_us-gaap_OtherExpenses_label_en-US" xlink:label="lab_us-gaap_OtherExpenses" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Other Expenses</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_OtherExpenses" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OtherExpenses"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_OtherExpenses" xlink:to="lab_us-gaap_OtherExpenses" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_ConcentrationRiskCreditRiskFinancialInstrumentMaximumExposure_7559b554-cbac-440d-b93d-7b907ff52d69_terseLabel_en-US" xlink:label="lab_us-gaap_ConcentrationRiskCreditRiskFinancialInstrumentMaximumExposure" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Amount of Exposure</link:label>
    <link:label id="lab_us-gaap_ConcentrationRiskCreditRiskFinancialInstrumentMaximumExposure_label_en-US" xlink:label="lab_us-gaap_ConcentrationRiskCreditRiskFinancialInstrumentMaximumExposure" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Concentration Risk, Credit Risk, Financial Instrument, Maximum Exposure</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_ConcentrationRiskCreditRiskFinancialInstrumentMaximumExposure" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ConcentrationRiskCreditRiskFinancialInstrumentMaximumExposure"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_ConcentrationRiskCreditRiskFinancialInstrumentMaximumExposure" xlink:to="lab_us-gaap_ConcentrationRiskCreditRiskFinancialInstrumentMaximumExposure" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CommonStockNumberOfSharesParValueAndOtherDisclosuresAbstract_31b11bab-a77c-48e9-b945-1e96312a4c2f_terseLabel_en-US" xlink:label="lab_us-gaap_CommonStockNumberOfSharesParValueAndOtherDisclosuresAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">PER SHARE INFORMATION:</link:label>
    <link:label id="lab_us-gaap_CommonStockNumberOfSharesParValueAndOtherDisclosuresAbstract_label_en-US" xlink:label="lab_us-gaap_CommonStockNumberOfSharesParValueAndOtherDisclosuresAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Common Stock, Number of Shares, Par Value and Other Disclosures [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CommonStockNumberOfSharesParValueAndOtherDisclosuresAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CommonStockNumberOfSharesParValueAndOtherDisclosuresAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CommonStockNumberOfSharesParValueAndOtherDisclosuresAbstract" xlink:to="lab_us-gaap_CommonStockNumberOfSharesParValueAndOtherDisclosuresAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LineOfCreditFacilityInterestRateAtPeriodEnd_74461eef-ae28-490f-9cce-3c4e13713ff6_terseLabel_en-US" xlink:label="lab_us-gaap_LineOfCreditFacilityInterestRateAtPeriodEnd" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest rate</link:label>
    <link:label id="lab_us-gaap_LineOfCreditFacilityInterestRateAtPeriodEnd_label_en-US" xlink:label="lab_us-gaap_LineOfCreditFacilityInterestRateAtPeriodEnd" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Line of Credit Facility, Interest Rate at Period End</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LineOfCreditFacilityInterestRateAtPeriodEnd" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LineOfCreditFacilityInterestRateAtPeriodEnd"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LineOfCreditFacilityInterestRateAtPeriodEnd" xlink:to="lab_us-gaap_LineOfCreditFacilityInterestRateAtPeriodEnd" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationFourPointFivePercentMember_16777702-736f-4f95-abe5-d49704211db3_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationFourPointFivePercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Four Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationFourPointFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationFourPointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Four Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationFourPointFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationFourPointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationFourPointFivePercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationFourPointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleRealizedLossExcludingOtherThanTemporaryImpairment_1b92a837-28cf-46e9-a439-6e77d9fd05c2_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleRealizedLossExcludingOtherThanTemporaryImpairment" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Available-for-sale securities Realizes Losses - adjustments to cost basis of certain securities</link:label>
    <link:label id="lab_us-gaap_DebtSecuritiesAvailableForSaleRealizedLossExcludingOtherThanTemporaryImpairment_label_en-US" xlink:label="lab_us-gaap_DebtSecuritiesAvailableForSaleRealizedLossExcludingOtherThanTemporaryImpairment" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Debt Securities, Available-for-sale, Realized Loss, Excluding Other-than-temporary Impairment</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DebtSecuritiesAvailableForSaleRealizedLossExcludingOtherThanTemporaryImpairment" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DebtSecuritiesAvailableForSaleRealizedLossExcludingOtherThanTemporaryImpairment"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DebtSecuritiesAvailableForSaleRealizedLossExcludingOtherThanTemporaryImpairment" xlink:to="lab_us-gaap_DebtSecuritiesAvailableForSaleRealizedLossExcludingOtherThanTemporaryImpairment" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_NotePurchaseAgreementDecember2019Member_278304d1-eb93-4b40-995f-e380ee248ed6_terseLabel_en-US" xlink:label="lab_efc_NotePurchaseAgreementDecember2019Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Note Purchase Agreement December 2019 [Member]</link:label>
    <link:label id="lab_efc_NotePurchaseAgreementDecember2019Member_label_en-US" xlink:label="lab_efc_NotePurchaseAgreementDecember2019Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Note Purchase Agreement December 2019 [Member]</link:label>
    <link:label id="lab_efc_NotePurchaseAgreementDecember2019Member_documentation_en-US" xlink:label="lab_efc_NotePurchaseAgreementDecember2019Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Note Purchase Agreement December 2019 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_NotePurchaseAgreementDecember2019Member" xlink:href="efc-20201231.xsd#efc_NotePurchaseAgreementDecember2019Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_NotePurchaseAgreementDecember2019Member" xlink:to="lab_efc_NotePurchaseAgreementDecember2019Member" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_A2050Member_f29c9e35-06d8-4c9f-b846-26d9f7d61ded_terseLabel_en-US" xlink:label="lab_efc_A2050Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2050</link:label>
    <link:label id="lab_efc_A2050Member_label_en-US" xlink:label="lab_efc_A2050Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2050 [Member]</link:label>
    <link:label id="lab_efc_A2050Member_documentation_en-US" xlink:label="lab_efc_A2050Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">2050</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_A2050Member" xlink:href="efc-20201231.xsd#efc_A2050Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_A2050Member" xlink:to="lab_efc_A2050Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentTypeAxis_05bbcdc7-4143-4e76-8c2b-392dff13cba1_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentTypeAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Type [Axis]</link:label>
    <link:label id="lab_us-gaap_InvestmentTypeAxis_label_en-US" xlink:label="lab_us-gaap_InvestmentTypeAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Type [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentTypeAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentTypeAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentTypeAxis" xlink:to="lab_us-gaap_InvestmentTypeAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InterestPaidNet_b99ace1e-bf5a-44bc-9344-9d41fe4c0e5a_terseLabel_en-US" xlink:label="lab_us-gaap_InterestPaidNet" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest paid</link:label>
    <link:label id="lab_us-gaap_InterestPaidNet_label_en-US" xlink:label="lab_us-gaap_InterestPaidNet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Interest Paid, Excluding Capitalized Interest, Operating Activities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InterestPaidNet" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InterestPaidNet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InterestPaidNet" xlink:to="lab_us-gaap_InterestPaidNet" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_NotesReduction_0f7a0c0a-e42f-4b95-b3e5-9eb09c609973_negatedTerseLabel_en-US" xlink:label="lab_us-gaap_NotesReduction" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Repayment of senior notes (non-cash)</link:label>
    <link:label id="lab_us-gaap_NotesReduction_label_en-US" xlink:label="lab_us-gaap_NotesReduction" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Notes Reduction</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_NotesReduction" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NotesReduction"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_NotesReduction" xlink:to="lab_us-gaap_NotesReduction" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ElizonAFG20181LLCMember_a63f8d34-fde8-456b-b998-57584e746040_terseLabel_en-US" xlink:label="lab_efc_ElizonAFG20181LLCMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Elizon AFG 2018-1 LLC [Member]</link:label>
    <link:label id="lab_efc_ElizonAFG20181LLCMember_label_en-US" xlink:label="lab_efc_ElizonAFG20181LLCMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Elizon AFG 2018-1 LLC [Member]</link:label>
    <link:label id="lab_efc_ElizonAFG20181LLCMember_documentation_en-US" xlink:label="lab_efc_ElizonAFG20181LLCMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Elizon AFG 2018-1 LLC [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ElizonAFG20181LLCMember" xlink:href="efc-20201231.xsd#efc_ElizonAFG20181LLCMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ElizonAFG20181LLCMember" xlink:to="lab_efc_ElizonAFG20181LLCMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_BorrowingsunderReverseRepurchaseAgreements_7bef3355-d200-4e18-858a-602258bb9d80_terseLabel_en-US" xlink:label="lab_efc_BorrowingsunderReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Borrowings under repurchase agreements</link:label>
    <link:label id="lab_efc_BorrowingsunderReverseRepurchaseAgreements_label_en-US" xlink:label="lab_efc_BorrowingsunderReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Borrowings under Reverse Repurchase Agreements</link:label>
    <link:label id="lab_efc_BorrowingsunderReverseRepurchaseAgreements_documentation_en-US" xlink:label="lab_efc_BorrowingsunderReverseRepurchaseAgreements" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash inflow from securities sold under agreements to repurchase.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_BorrowingsunderReverseRepurchaseAgreements" xlink:href="efc-20201231.xsd#efc_BorrowingsunderReverseRepurchaseAgreements"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_BorrowingsunderReverseRepurchaseAgreements" xlink:to="lab_efc_BorrowingsunderReverseRepurchaseAgreements" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Securitizationclosingdate_647b83b6-6c65-4ea7-a03f-91e9781c41bc_terseLabel_en-US" xlink:label="lab_efc_Securitizationclosingdate" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">CLO Closing Date</link:label>
    <link:label id="lab_efc_Securitizationclosingdate_label_en-US" xlink:label="lab_efc_Securitizationclosingdate" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization closing date</link:label>
    <link:label id="lab_efc_Securitizationclosingdate_documentation_en-US" xlink:label="lab_efc_Securitizationclosingdate" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization closing date (settle date).</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Securitizationclosingdate" xlink:href="efc-20201231.xsd#efc_Securitizationclosingdate"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Securitizationclosingdate" xlink:to="lab_efc_Securitizationclosingdate" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_SecurityExchangeName_e696c948-fb7c-452f-abca-ae9cb48dcd8a_terseLabel_en-US" xlink:label="lab_dei_SecurityExchangeName" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Security Exchange Name</link:label>
    <link:label id="lab_dei_SecurityExchangeName_label_en-US" xlink:label="lab_dei_SecurityExchangeName" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Security Exchange Name</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_SecurityExchangeName" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_SecurityExchangeName"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_SecurityExchangeName" xlink:to="lab_dei_SecurityExchangeName" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Companysshareofloanssold_990b01c1-f3c6-4c31-b665-07edf3c25601_terseLabel_en-US" xlink:label="lab_efc_Companysshareofloanssold" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Company's share of loans sold</link:label>
    <link:label id="lab_efc_Companysshareofloanssold_label_en-US" xlink:label="lab_efc_Companysshareofloanssold" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Company's share of loans sold</link:label>
    <link:label id="lab_efc_Companysshareofloanssold_documentation_en-US" xlink:label="lab_efc_Companysshareofloanssold" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Percentage of aggregate unpaid principal balance of consumer loans sold by the Company to Issuer in securitization transaction.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Companysshareofloanssold" xlink:href="efc-20201231.xsd#efc_Companysshareofloanssold"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Companysshareofloanssold" xlink:to="lab_efc_Companysshareofloanssold" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseLineItems_a4a28071-e0d7-4c95-8c67-6f14010c8336_terseLabel_en-US" xlink:label="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets Sold under Agreements to Repurchase [Line Items]</link:label>
    <link:label id="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseLineItems_label_en-US" xlink:label="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets Sold under Agreements to Repurchase [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_AssetsSoldUnderAgreementsToRepurchaseLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AssetsSoldUnderAgreementsToRepurchaseLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_AssetsSoldUnderAgreementsToRepurchaseLineItems" xlink:to="lab_us-gaap_AssetsSoldUnderAgreementsToRepurchaseLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_MarketValueUnderlying_6ed36fc9-fd8e-42b9-bf5e-4e95ee077977_terseLabel_en-US" xlink:label="lab_efc_MarketValueUnderlying" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Market Value Underlying</link:label>
    <link:label id="lab_efc_MarketValueUnderlying_label_en-US" xlink:label="lab_efc_MarketValueUnderlying" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Market Value Underlying</link:label>
    <link:label id="lab_efc_MarketValueUnderlying_documentation_en-US" xlink:label="lab_efc_MarketValueUnderlying" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Market value (notional amount times price at the end of the period) of the security underyling the derivative.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_MarketValueUnderlying" xlink:href="efc-20201231.xsd#efc_MarketValueUnderlying"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_MarketValueUnderlying" xlink:to="lab_efc_MarketValueUnderlying" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_SecuritizationTransactionsTable_ce422fe8-39c9-48c8-909a-719d26f6012f_terseLabel_en-US" xlink:label="lab_efc_SecuritizationTransactionsTable" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization Transactions [Table]</link:label>
    <link:label id="lab_efc_SecuritizationTransactionsTable_label_en-US" xlink:label="lab_efc_SecuritizationTransactionsTable" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization Transactions [Table]</link:label>
    <link:label id="lab_efc_SecuritizationTransactionsTable_documentation_en-US" xlink:label="lab_efc_SecuritizationTransactionsTable" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Securitization Transactions [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_SecuritizationTransactionsTable" xlink:href="efc-20201231.xsd#efc_SecuritizationTransactionsTable"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_SecuritizationTransactionsTable" xlink:to="lab_efc_SecuritizationTransactionsTable" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember_a3f7c9e5-d33d-4971-8e7e-b4c43488c759_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Three Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Three Point Five Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association Pools (Thirty Year) Three Point Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociationPoolsThirtyYearThreePointFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_StockRepurchaseProgramNumberOfSharesAuthorizedToBeRepurchased_907a1671-a7b5-4908-b932-29e8abb3d07c_terseLabel_en-US" xlink:label="lab_us-gaap_StockRepurchaseProgramNumberOfSharesAuthorizedToBeRepurchased" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Number of shares authorized to be repurchased (in shares)</link:label>
    <link:label id="lab_us-gaap_StockRepurchaseProgramNumberOfSharesAuthorizedToBeRepurchased_label_en-US" xlink:label="lab_us-gaap_StockRepurchaseProgramNumberOfSharesAuthorizedToBeRepurchased" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Stock Repurchase Program, Number of Shares Authorized to be Repurchased</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_StockRepurchaseProgramNumberOfSharesAuthorizedToBeRepurchased" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockRepurchaseProgramNumberOfSharesAuthorizedToBeRepurchased"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_StockRepurchaseProgramNumberOfSharesAuthorizedToBeRepurchased" xlink:to="lab_us-gaap_StockRepurchaseProgramNumberOfSharesAuthorizedToBeRepurchased" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_LongtermDebtTypeAxis_7f045f76-94fe-4eb1-bf25-72b4e14418b6_terseLabel_en-US" xlink:label="lab_us-gaap_LongtermDebtTypeAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Type [Axis]</link:label>
    <link:label id="lab_us-gaap_LongtermDebtTypeAxis_label_en-US" xlink:label="lab_us-gaap_LongtermDebtTypeAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Long-term Debt, Type [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_LongtermDebtTypeAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LongtermDebtTypeAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_LongtermDebtTypeAxis" xlink:to="lab_us-gaap_LongtermDebtTypeAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GreaterthanthreeyearslessthansevenyearsMember_a4d1afb4-98cf-4139-97a4-fbf4da35cc6d_terseLabel_en-US" xlink:label="lab_efc_GreaterthanthreeyearslessthansevenyearsMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Greater than three years less than seven years [Member]</link:label>
    <link:label id="lab_efc_GreaterthanthreeyearslessthansevenyearsMember_label_en-US" xlink:label="lab_efc_GreaterthanthreeyearslessthansevenyearsMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Greater than three years less than seven years [Member]</link:label>
    <link:label id="lab_efc_GreaterthanthreeyearslessthansevenyearsMember_documentation_en-US" xlink:label="lab_efc_GreaterthanthreeyearslessthansevenyearsMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Greater than three years less than seven years [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GreaterthanthreeyearslessthansevenyearsMember" xlink:href="efc-20201231.xsd#efc_GreaterthanthreeyearslessthansevenyearsMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GreaterthanthreeyearslessthansevenyearsMember" xlink:to="lab_efc_GreaterthanthreeyearslessthansevenyearsMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociationFivePercentMember_210286ee-752f-40f6-8ce4-6e1cb386842e_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationFivePercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Five Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociationFivePercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociationFivePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association Five Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociationFivePercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociationFivePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociationFivePercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociationFivePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquityComponentDomain_6b0a6fa2-9dfc-484a-bb08-63d135aa378d_terseLabel_en-US" xlink:label="lab_us-gaap_EquityComponentDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Component [Domain]</link:label>
    <link:label id="lab_us-gaap_EquityComponentDomain_label_en-US" xlink:label="lab_us-gaap_EquityComponentDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Component [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquityComponentDomain" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquityComponentDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquityComponentDomain" xlink:to="lab_us-gaap_EquityComponentDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_dei_DocumentTransitionReport_cc72340a-ec1e-4fb3-8ac5-ceb485fe4379_terseLabel_en-US" xlink:label="lab_dei_DocumentTransitionReport" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Document Transition Report</link:label>
    <link:label id="lab_dei_DocumentTransitionReport_label_en-US" xlink:label="lab_dei_DocumentTransitionReport" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Document Transition Report</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_dei_DocumentTransitionReport" xlink:href="https://xbrl.sec.gov/dei/2019/dei-2019-01-31.xsd#dei_DocumentTransitionReport"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_dei_DocumentTransitionReport" xlink:to="lab_dei_DocumentTransitionReport" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_RealEstateAcquiredThroughForeclosure_31946052-ac5e-4a11-a773-cfd5f22eb5e9_terseLabel_en-US" xlink:label="lab_us-gaap_RealEstateAcquiredThroughForeclosure" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real estate owned</link:label>
    <link:label id="lab_us-gaap_RealEstateAcquiredThroughForeclosure_label_en-US" xlink:label="lab_us-gaap_RealEstateAcquiredThroughForeclosure" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_RealEstateAcquiredThroughForeclosure" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RealEstateAcquiredThroughForeclosure"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_RealEstateAcquiredThroughForeclosure" xlink:to="lab_us-gaap_RealEstateAcquiredThroughForeclosure" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeCollateralRightToReclaimCash_37699974-9bc9-427f-a67c-ebbe55ecd1e5_verboseLabel_en-US" xlink:label="lab_us-gaap_DerivativeCollateralRightToReclaimCash" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash Collateral (Received) Pledged</link:label>
    <link:label id="lab_us-gaap_DerivativeCollateralRightToReclaimCash_label_en-US" xlink:label="lab_us-gaap_DerivativeCollateralRightToReclaimCash" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative, Collateral, Right to Reclaim Cash</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeCollateralRightToReclaimCash" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeCollateralRightToReclaimCash"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeCollateralRightToReclaimCash" xlink:to="lab_us-gaap_DerivativeCollateralRightToReclaimCash" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociation15Year3Point00PercentMember_167efc8b-6c9f-4175-a99f-a3a3d572b3b3_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation15Year3Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (15 Year) 3 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation15Year3Point00PercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation15Year3Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (15 Year) 3 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation15Year3Point00PercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation15Year3Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (15 Year) 3 Point 00 Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociation15Year3Point00PercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociation15Year3Point00PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociation15Year3Point00PercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociation15Year3Point00PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InvestmentInLoanOriginatorMember_b8e402a8-52e6-430f-a9ad-080766ec408e_terseLabel_en-US" xlink:label="lab_efc_InvestmentInLoanOriginatorMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment in Loan Originator</link:label>
    <link:label id="lab_efc_InvestmentInLoanOriginatorMember_label_en-US" xlink:label="lab_efc_InvestmentInLoanOriginatorMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment in Loan Originator [Member]</link:label>
    <link:label id="lab_efc_InvestmentInLoanOriginatorMember_documentation_en-US" xlink:label="lab_efc_InvestmentInLoanOriginatorMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment in Loan Originator</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InvestmentInLoanOriginatorMember" xlink:href="efc-20201231.xsd#efc_InvestmentInLoanOriginatorMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InvestmentInLoanOriginatorMember" xlink:to="lab_efc_InvestmentInLoanOriginatorMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Cashcollateralpostedforsecuritiessoldunderagreementstorepurchase_15f0a205-eb64-4e16-b021-4e04bc6b881c_terseLabel_en-US" xlink:label="lab_efc_Cashcollateralpostedforsecuritiessoldunderagreementstorepurchase" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash collateral posted for securities sold under agreements to repurchase</link:label>
    <link:label id="lab_efc_Cashcollateralpostedforsecuritiessoldunderagreementstorepurchase_label_en-US" xlink:label="lab_efc_Cashcollateralpostedforsecuritiessoldunderagreementstorepurchase" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash collateral posted for securities sold under agreements to repurchase</link:label>
    <link:label id="lab_efc_Cashcollateralpostedforsecuritiessoldunderagreementstorepurchase_documentation_en-US" xlink:label="lab_efc_Cashcollateralpostedforsecuritiessoldunderagreementstorepurchase" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash collateral posted for securities sold under agreements to repurchase.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Cashcollateralpostedforsecuritiessoldunderagreementstorepurchase" xlink:href="efc-20201231.xsd#efc_Cashcollateralpostedforsecuritiessoldunderagreementstorepurchase"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Cashcollateralpostedforsecuritiessoldunderagreementstorepurchase" xlink:to="lab_efc_Cashcollateralpostedforsecuritiessoldunderagreementstorepurchase" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CreditRiskDerivativesAtFairValueNet_b46b8f1b-35de-4e61-b948-2f46eb0b3832_terseLabel_en-US" xlink:label="lab_us-gaap_CreditRiskDerivativesAtFairValueNet" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Risk Derivatives, at Fair Value, Net</link:label>
    <link:label id="lab_us-gaap_CreditRiskDerivativesAtFairValueNet_label_en-US" xlink:label="lab_us-gaap_CreditRiskDerivativesAtFairValueNet" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Risk Derivatives, at Fair Value, Net</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CreditRiskDerivativesAtFairValueNet" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CreditRiskDerivativesAtFairValueNet"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CreditRiskDerivativesAtFairValueNet" xlink:to="lab_us-gaap_CreditRiskDerivativesAtFairValueNet" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureRealizedGainsLossesOnDisposal_cf2c7c62-ac6b-4d63-bef1-8cf0e5814fe0_terseLabel_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureRealizedGainsLossesOnDisposal" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized gains (losses) on real estate owned, net</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureRealizedGainsLossesOnDisposal_c9301a7e-d8e5-4f6d-b7ae-8425f45b0006_negatedTerseLabel_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureRealizedGainsLossesOnDisposal" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Realized (gains) losses on real estate owned, net</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureRealizedGainsLossesOnDisposal_label_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureRealizedGainsLossesOnDisposal" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure, Realized Gains (Losses) On Disposal</link:label>
    <link:label id="lab_efc_RealEstateAcquiredThroughForeclosureRealizedGainsLossesOnDisposal_documentation_en-US" xlink:label="lab_efc_RealEstateAcquiredThroughForeclosureRealizedGainsLossesOnDisposal" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Real Estate Acquired Through Foreclosure, Realized Gains (Losses) On Disposal</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_RealEstateAcquiredThroughForeclosureRealizedGainsLossesOnDisposal" xlink:href="efc-20201231.xsd#efc_RealEstateAcquiredThroughForeclosureRealizedGainsLossesOnDisposal"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_RealEstateAcquiredThroughForeclosureRealizedGainsLossesOnDisposal" xlink:to="lab_efc_RealEstateAcquiredThroughForeclosureRealizedGainsLossesOnDisposal" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DeferredTaxAssetsGross_f4310570-5cd9-45ad-a56a-d3714a1fd025_terseLabel_en-US" xlink:label="lab_us-gaap_DeferredTaxAssetsGross" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Tax Assets, Gross</link:label>
    <link:label id="lab_us-gaap_DeferredTaxAssetsGross_label_en-US" xlink:label="lab_us-gaap_DeferredTaxAssetsGross" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Deferred Tax Assets, Gross</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DeferredTaxAssetsGross" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DeferredTaxAssetsGross"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DeferredTaxAssetsGross" xlink:to="lab_us-gaap_DeferredTaxAssetsGross" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year3Point50PercentMember_eb7d5b08-8a99-46ce-9d66-e4170cdeab26_terseLabel_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year3Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 3 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year3Point50PercentMember_label_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year3Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 3 Point 50 Percent [Member]</link:label>
    <link:label id="lab_efc_FederalNationalMortgageAssociation30Year3Point50PercentMember_documentation_en-US" xlink:label="lab_efc_FederalNationalMortgageAssociation30Year3Point50PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal National Mortgage Association (30 Year) 3.50%</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalNationalMortgageAssociation30Year3Point50PercentMember" xlink:href="efc-20201231.xsd#efc_FederalNationalMortgageAssociation30Year3Point50PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalNationalMortgageAssociation30Year3Point50PercentMember" xlink:to="lab_efc_FederalNationalMortgageAssociation30Year3Point50PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_srt_ConsolidatedEntitiesDomain_3f778d0c-6fc1-4c09-89ad-d770438dbcc3_terseLabel_en-US" xlink:label="lab_srt_ConsolidatedEntitiesDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidated Entities [Domain]</link:label>
    <link:label id="lab_srt_ConsolidatedEntitiesDomain_label_en-US" xlink:label="lab_srt_ConsolidatedEntitiesDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidated Entities [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_srt_ConsolidatedEntitiesDomain" xlink:href="http://xbrl.fasb.org/srt/2019/elts/srt-2019-01-31.xsd#srt_ConsolidatedEntitiesDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_srt_ConsolidatedEntitiesDomain" xlink:to="lab_srt_ConsolidatedEntitiesDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_Proceedsfromissuanceofsecuritizeddebt_bcd67275-9561-4c5c-8a50-619d9864dd0f_terseLabel_en-US" xlink:label="lab_efc_Proceedsfromissuanceofsecuritizeddebt" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from issuance of securitized debt</link:label>
    <link:label id="lab_efc_Proceedsfromissuanceofsecuritizeddebt_label_en-US" xlink:label="lab_efc_Proceedsfromissuanceofsecuritizeddebt" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Proceeds from issuance of securitized debt</link:label>
    <link:label id="lab_efc_Proceedsfromissuanceofsecuritizeddebt_documentation_en-US" xlink:label="lab_efc_Proceedsfromissuanceofsecuritizeddebt" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Cash inflow from issuance of collateralized borrowings.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_Proceedsfromissuanceofsecuritizeddebt" xlink:href="efc-20201231.xsd#efc_Proceedsfromissuanceofsecuritizeddebt"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_Proceedsfromissuanceofsecuritizeddebt" xlink:to="lab_efc_Proceedsfromissuanceofsecuritizeddebt" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_InvestmentCreditRatingAxis_0f2b422a-1323-4616-bb9b-1c38d3f0f4a0_terseLabel_en-US" xlink:label="lab_efc_InvestmentCreditRatingAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Credit Rating [Axis]</link:label>
    <link:label id="lab_efc_InvestmentCreditRatingAxis_label_en-US" xlink:label="lab_efc_InvestmentCreditRatingAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Credit Rating [Axis]</link:label>
    <link:label id="lab_efc_InvestmentCreditRatingAxis_documentation_en-US" xlink:label="lab_efc_InvestmentCreditRatingAxis" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Information by investment credit rating.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_InvestmentCreditRatingAxis" xlink:href="efc-20201231.xsd#efc_InvestmentCreditRatingAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_InvestmentCreditRatingAxis" xlink:to="lab_efc_InvestmentCreditRatingAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquitySecuritiesByIndustryAxis_9d5f4b39-a7f1-4913-8692-d999ca829428_terseLabel_en-US" xlink:label="lab_us-gaap_EquitySecuritiesByIndustryAxis" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Industry Sector [Axis]</link:label>
    <link:label id="lab_us-gaap_EquitySecuritiesByIndustryAxis_label_en-US" xlink:label="lab_us-gaap_EquitySecuritiesByIndustryAxis" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Industry Sector [Axis]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquitySecuritiesByIndustryAxis" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquitySecuritiesByIndustryAxis"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquitySecuritiesByIndustryAxis" xlink:to="lab_us-gaap_EquitySecuritiesByIndustryAxis" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_VestDecember112020Member_60f996d8-0ed8-45b1-b343-92e74971422a_terseLabel_en-US" xlink:label="lab_efc_VestDecember112020Member" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 11, 2020 [Member]</link:label>
    <link:label id="lab_efc_VestDecember112020Member_label_en-US" xlink:label="lab_efc_VestDecember112020Member" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 11, 2020 [Member]</link:label>
    <link:label id="lab_efc_VestDecember112020Member_documentation_en-US" xlink:label="lab_efc_VestDecember112020Member" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Vest December 11, 2020 [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_VestDecember112020Member" xlink:href="efc-20201231.xsd#efc_VestDecember112020Member"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_VestDecember112020Member" xlink:to="lab_efc_VestDecember112020Member" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_CreditDerivativesLineItems_c294b81d-5803-4729-ae73-bbacd6c1a6cd_terseLabel_en-US" xlink:label="lab_us-gaap_CreditDerivativesLineItems" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Derivatives [Line Items]</link:label>
    <link:label id="lab_us-gaap_CreditDerivativesLineItems_label_en-US" xlink:label="lab_us-gaap_CreditDerivativesLineItems" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Derivatives [Line Items]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_CreditDerivativesLineItems" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CreditDerivativesLineItems"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_CreditDerivativesLineItems" xlink:to="lab_us-gaap_CreditDerivativesLineItems" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersIntoLevel3_c7b92b56-c8c3-495b-8d75-c200fd0d534b_terseLabel_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersIntoLevel3" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Liabilities, Transfers into Level 3</link:label>
    <link:label id="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersIntoLevel3_label_en-US" xlink:label="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersIntoLevel3" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair Value, Measurement with Unobservable Inputs Reconciliation, Liability, Transfers Into Level 3</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersIntoLevel3" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersIntoLevel3"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersIntoLevel3" xlink:to="lab_us-gaap_FairValueMeasurementWithUnobservableInputsReconciliationLiabilityTransfersIntoLevel3" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_CreditDefaultSwapAssetBackedIndicesMember_693dc038-5be7-4aaa-8f90-1023009fa390_terseLabel_en-US" xlink:label="lab_efc_CreditDefaultSwapAssetBackedIndicesMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit default swaps on asset-backed indices</link:label>
    <link:label id="lab_efc_CreditDefaultSwapAssetBackedIndicesMember_09125554-5902-4c3f-bcc6-195078037b5a_verboseLabel_en-US" xlink:label="lab_efc_CreditDefaultSwapAssetBackedIndicesMember" xlink:role="http://www.xbrl.org/2003/role/verboseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit default swaps on corporate bond indices</link:label>
    <link:label id="lab_efc_CreditDefaultSwapAssetBackedIndicesMember_label_en-US" xlink:label="lab_efc_CreditDefaultSwapAssetBackedIndicesMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swap, Asset-Backed Indices [Member]</link:label>
    <link:label id="lab_efc_CreditDefaultSwapAssetBackedIndicesMember_documentation_en-US" xlink:label="lab_efc_CreditDefaultSwapAssetBackedIndicesMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Credit Default Swap, Asset-Backed Indices [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_CreditDefaultSwapAssetBackedIndicesMember" xlink:href="efc-20201231.xsd#efc_CreditDefaultSwapAssetBackedIndicesMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_CreditDefaultSwapAssetBackedIndicesMember" xlink:to="lab_efc_CreditDefaultSwapAssetBackedIndicesMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_U.S.TreasuryNote3.13PercentMaturityDate202811MemberDomain_ad037493-39b7-4ffb-8e65-4e9a44cfb7b1_terseLabel_en-US" xlink:label="lab_efc_U.S.TreasuryNote3.13PercentMaturityDate202811MemberDomain" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 3.13 Percent, Maturity Date 2028 11 [Member] [Domain]</link:label>
    <link:label id="lab_efc_U.S.TreasuryNote3.13PercentMaturityDate202811MemberDomain_label_en-US" xlink:label="lab_efc_U.S.TreasuryNote3.13PercentMaturityDate202811MemberDomain" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 3.13 Percent, Maturity Date 2028 11 [Member] [Domain]</link:label>
    <link:label id="lab_efc_U.S.TreasuryNote3.13PercentMaturityDate202811MemberDomain_documentation_en-US" xlink:label="lab_efc_U.S.TreasuryNote3.13PercentMaturityDate202811MemberDomain" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">U.S. Treasury Note, 3.13 Percent, Maturity Date 2028 11 [Member] [Domain]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_U.S.TreasuryNote3.13PercentMaturityDate202811MemberDomain" xlink:href="efc-20201231.xsd#efc_U.S.TreasuryNote3.13PercentMaturityDate202811MemberDomain"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_U.S.TreasuryNote3.13PercentMaturityDate202811MemberDomain" xlink:to="lab_efc_U.S.TreasuryNote3.13PercentMaturityDate202811MemberDomain" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_MaturityUpTo30DaysMember_57674897-6996-40ab-814a-deda266cf5b3_terseLabel_en-US" xlink:label="lab_us-gaap_MaturityUpTo30DaysMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">30 Days or Less</link:label>
    <link:label id="lab_us-gaap_MaturityUpTo30DaysMember_label_en-US" xlink:label="lab_us-gaap_MaturityUpTo30DaysMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Maturity Less than 30 Days [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_MaturityUpTo30DaysMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MaturityUpTo30DaysMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_MaturityUpTo30DaysMember" xlink:to="lab_us-gaap_MaturityUpTo30DaysMember" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_FinancialInstrumentsFinancialAssetsBalanceSheetGroupingsAbstract_93e83107-3ae6-4bc0-a4fa-57f7fdbf46eb_terseLabel_en-US" xlink:label="lab_us-gaap_FinancialInstrumentsFinancialAssetsBalanceSheetGroupingsAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Assets:</link:label>
    <link:label id="lab_us-gaap_FinancialInstrumentsFinancialAssetsBalanceSheetGroupingsAbstract_label_en-US" xlink:label="lab_us-gaap_FinancialInstrumentsFinancialAssetsBalanceSheetGroupingsAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Financial Instruments, Financial Assets, Balance Sheet Groupings [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_FinancialInstrumentsFinancialAssetsBalanceSheetGroupingsAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinancialInstrumentsFinancialAssetsBalanceSheetGroupingsAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_FinancialInstrumentsFinancialAssetsBalanceSheetGroupingsAbstract" xlink:to="lab_us-gaap_FinancialInstrumentsFinancialAssetsBalanceSheetGroupingsAbstract" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationAssets_b68e3f9f-8b69-4ce0-805a-5187621f4084_terseLabel_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationAssets" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Assets</link:label>
    <link:label id="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationAssets_label_en-US" xlink:label="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationAssets" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Equity Method Investment, Summarized Financial Information, Assets</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationAssets" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_EquityMethodInvestmentSummarizedFinancialInformationAssets"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationAssets" xlink:to="lab_us-gaap_EquityMethodInvestmentSummarizedFinancialInformationAssets" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_ScheduleofDerivativeWarrantContractsTableTextBlock_266bced0-2e57-4249-b60d-94ad5a59cf9a_terseLabel_en-US" xlink:label="lab_efc_ScheduleofDerivativeWarrantContractsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Derivative Warrant Contracts [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleofDerivativeWarrantContractsTableTextBlock_label_en-US" xlink:label="lab_efc_ScheduleofDerivativeWarrantContractsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Schedule of Derivative Warrant Contracts [Table Text Block]</link:label>
    <link:label id="lab_efc_ScheduleofDerivativeWarrantContractsTableTextBlock_documentation_en-US" xlink:label="lab_efc_ScheduleofDerivativeWarrantContractsTableTextBlock" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">[Table Text Block] for Schedule of Derivative Warrant Contracts [Table]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_ScheduleofDerivativeWarrantContractsTableTextBlock" xlink:href="efc-20201231.xsd#efc_ScheduleofDerivativeWarrantContractsTableTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_ScheduleofDerivativeWarrantContractsTableTextBlock" xlink:to="lab_efc_ScheduleofDerivativeWarrantContractsTableTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_PreviousAccountingGuidanceMember_60e01f06-c26f-4a65-803a-9fb739247504_terseLabel_en-US" xlink:label="lab_us-gaap_PreviousAccountingGuidanceMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Previous Accounting Guidance [Member]</link:label>
    <link:label id="lab_us-gaap_PreviousAccountingGuidanceMember_label_en-US" xlink:label="lab_us-gaap_PreviousAccountingGuidanceMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Previous Accounting Guidance [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_PreviousAccountingGuidanceMember" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PreviousAccountingGuidanceMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_PreviousAccountingGuidanceMember" xlink:to="lab_us-gaap_PreviousAccountingGuidanceMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_PaymentsToBrokersInvestingActivities_926382c2-ced5-4e2c-9c61-2fc49c14ff13_negatedTerseLabel_en-US" xlink:label="lab_efc_PaymentsToBrokersInvestingActivities" xlink:role="http://www.xbrl.org/2009/role/negatedTerseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Due to brokers, net</link:label>
    <link:label id="lab_efc_PaymentsToBrokersInvestingActivities_label_en-US" xlink:label="lab_efc_PaymentsToBrokersInvestingActivities" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments To Brokers, Investing Activities</link:label>
    <link:label id="lab_efc_PaymentsToBrokersInvestingActivities_documentation_en-US" xlink:label="lab_efc_PaymentsToBrokersInvestingActivities" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Payments To Brokers, Investing Activities</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_PaymentsToBrokersInvestingActivities" xlink:href="efc-20201231.xsd#efc_PaymentsToBrokersInvestingActivities"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_PaymentsToBrokersInvestingActivities" xlink:to="lab_efc_PaymentsToBrokersInvestingActivities" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_VariableInterestEntityDisclosureTextBlock_828a945e-e855-495e-aff1-a3f52ab1219d_terseLabel_en-US" xlink:label="lab_us-gaap_VariableInterestEntityDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Consolidated VIEs</link:label>
    <link:label id="lab_us-gaap_VariableInterestEntityDisclosureTextBlock_label_en-US" xlink:label="lab_us-gaap_VariableInterestEntityDisclosureTextBlock" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Variable Interest Entity Disclosure [Text Block]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_VariableInterestEntityDisclosureTextBlock" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_VariableInterestEntityDisclosureTextBlock"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_VariableInterestEntityDisclosureTextBlock" xlink:to="lab_us-gaap_VariableInterestEntityDisclosureTextBlock" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_DerivativeFairValueOfDerivativeAsset_51851885-aa58-498a-b486-0b78343d191c_terseLabel_en-US" xlink:label="lab_us-gaap_DerivativeFairValueOfDerivativeAsset" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Asset, Fair Value, Gross Asset</link:label>
    <link:label id="lab_us-gaap_DerivativeFairValueOfDerivativeAsset_label_en-US" xlink:label="lab_us-gaap_DerivativeFairValueOfDerivativeAsset" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Derivative Asset, Fair Value, Gross Asset</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_DerivativeFairValueOfDerivativeAsset" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeFairValueOfDerivativeAsset"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_DerivativeFairValueOfDerivativeAsset" xlink:to="lab_us-gaap_DerivativeFairValueOfDerivativeAsset" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year4Point00PercentMember_62ae8506-0f75-4d2b-bbd2-db44c9644aca_terseLabel_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year4Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 4 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year4Point00PercentMember_label_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year4Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 4 Point 00 Percent [Member]</link:label>
    <link:label id="lab_efc_GovernmentNationalMortgageAssociation30Year4Point00PercentMember_documentation_en-US" xlink:label="lab_efc_GovernmentNationalMortgageAssociation30Year4Point00PercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Government National Mortgage Association (30 Year) 4 Point 00 Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_GovernmentNationalMortgageAssociation30Year4Point00PercentMember" xlink:href="efc-20201231.xsd#efc_GovernmentNationalMortgageAssociation30Year4Point00PercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_GovernmentNationalMortgageAssociation30Year4Point00PercentMember" xlink:to="lab_efc_GovernmentNationalMortgageAssociation30Year4Point00PercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePercentMember_3dd897b4-2a7d-419d-af1c-1cd4a24b3743_terseLabel_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePercentMember" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Three Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePercentMember_label_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePercentMember" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Three Percent [Member]</link:label>
    <link:label id="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePercentMember_documentation_en-US" xlink:label="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePercentMember" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Federal Home Loan Mortgage Corporation Pools (Thirty Year) Three Percent [Member]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePercentMember" xlink:href="efc-20201231.xsd#efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePercentMember"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePercentMember" xlink:to="lab_efc_FederalHomeLoanMortgageCorporationPoolsThirtyYearThreePercentMember" xlink:type="arc" order="1"/>
    <link:label id="lab_efc_UnencumberedInvestmentsHeldInCustody_418fc6e8-4d32-4d7d-bd96-0c26a014a0c7_terseLabel_en-US" xlink:label="lab_efc_UnencumberedInvestmentsHeldInCustody" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unencumbered investments held in custody</link:label>
    <link:label id="lab_efc_UnencumberedInvestmentsHeldInCustody_label_en-US" xlink:label="lab_efc_UnencumberedInvestmentsHeldInCustody" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Unencumbered Investments Held In Custody</link:label>
    <link:label id="lab_efc_UnencumberedInvestmentsHeldInCustody_documentation_en-US" xlink:label="lab_efc_UnencumberedInvestmentsHeldInCustody" xlink:role="http://www.xbrl.org/2003/role/documentation" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Fair value as of the balance sheet date of securities that are free and clear of any encumbrances such as reverse repurchase agreements.</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_efc_UnencumberedInvestmentsHeldInCustody" xlink:href="efc-20201231.xsd#efc_UnencumberedInvestmentsHeldInCustody"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_efc_UnencumberedInvestmentsHeldInCustody" xlink:to="lab_efc_UnencumberedInvestmentsHeldInCustody" xlink:type="arc" order="1"/>
    <link:label id="lab_us-gaap_InvestmentIncomeNetAbstract_77742b03-73e4-45ff-9869-89cfc5d5ef4e_terseLabel_en-US" xlink:label="lab_us-gaap_InvestmentIncomeNetAbstract" xlink:role="http://www.xbrl.org/2003/role/terseLabel" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Net Interest Income</link:label>
    <link:label id="lab_us-gaap_InvestmentIncomeNetAbstract_label_en-US" xlink:label="lab_us-gaap_InvestmentIncomeNetAbstract" xlink:role="http://www.xbrl.org/2003/role/label" xlink:type="resource" xmlns:xml="http://www.w3.org/XML/1998/namespace" xml:lang="en-US">Investment Income, Net [Abstract]</link:label>
    <link:loc xlink:type="locator" xlink:label="loc_us-gaap_InvestmentIncomeNetAbstract" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InvestmentIncomeNetAbstract"/>
    <link:labelArc xlink:arcrole="http://www.xbrl.org/2003/arcrole/concept-label" xlink:from="loc_us-gaap_InvestmentIncomeNetAbstract" xlink:to="lab_us-gaap_InvestmentIncomeNetAbstract" xlink:type="arc" order="1"/>
  </link:labelLink>
</link:linkbase>
