<?xml version="1.0" encoding="us-ascii"?><InstanceReport xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><Version>2.4.0.8</Version><ReportLongName>125 - Disclosure - Exploration Expense</ReportLongName><DisplayLabelColumn>true</DisplayLabelColumn><ShowElementNames>false</ShowElementNames><RoundingOption /><HasEmbeddedReports>false</HasEmbeddedReports><Columns><Column FlagID="0"><Id>1</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><CurrencyCode /><FootnoteIndexer /><hasSegments>false</hasSegments><hasScenarios>false</hasScenarios><MCU><KeyName /><CurrencySymbol /><contextRef><ContextID>eol_PE742405--138-K-0014_STD_181_20130630_0</ContextID><EntitySchema>http://www.sec.gov/CIK</EntitySchema><EntityValue>0001397516</EntityValue><PeriodDisplayName /><PeriodType>duration</PeriodType><PeriodStartDate>2013-01-01T00:00:00</PeriodStartDate><PeriodEndDate>2013-06-30T00:00:00</PeriodEndDate><Segments /><Scenarios /></contextRef><UPS /><CurrencyCode /><OriginalCurrencyCode /></MCU><CurrencySymbol /><Labels><Label Key="CalendarSupplement" Id="0" Label="6 Months Ended" /><Label Key="Calendar" Id="1" Label="Jun. 30, 2013" /></Labels></Column></Columns><Rows><Row FlagID="0"><Id>1</Id><IsAbstractGroupTitle>false</IsAbstractGroupTitle><LabelSeparator>

</LabelSeparator><Level>4</Level><ElementName>rexx_ExplorationExpenseDisclosureTextBlock</ElementName><ElementPrefix>rexx_</ElementPrefix><IsBaseElement>false</IsBaseElement><BalanceType>na</BalanceType><PeriodType>duration</PeriodType><IsReportTitle>false</IsReportTitle><IsSegmentTitle>false</IsSegmentTitle><IsCalendarTitle>false</IsCalendarTitle><IsEquityPrevioslyReportedAsRow>false</IsEquityPrevioslyReportedAsRow><IsEquityAdjustmentRow>false</IsEquityAdjustmentRow><IsBeginningBalance>false</IsBeginningBalance><IsEndingBalance>false</IsEndingBalance><IsReverseSign>false</IsReverseSign><PreferredLabelRole>terseLabel</PreferredLabelRole><FootnoteIndexer /><Cells><Cell FlagID="0" ContextID="eol_PE742405--138-K-0014_STD_181_20130630_0" UnitID=""><Id>1</Id><IsNumeric>false</IsNumeric><IsRatio>false</IsRatio><DisplayZeroAsNone>false</DisplayZeroAsNone><NumericAmount>0</NumericAmount><RoundedNumericAmount>0</RoundedNumericAmount><NonNumbericText>&lt;div&gt;
 &lt;p style="MARGIN-TOP: 18px; MARGIN-BOTTOM: 0px"&gt;&lt;font style="FONT-FAMILY: Times New Roman" size="2"&gt;&lt;b&gt;18. EXPLORATION
 EXPENSE&lt;/b&gt;&lt;/font&gt;&lt;/p&gt;
 &lt;p style="MARGIN-TOP: 6px; TEXT-INDENT: 4%; MARGIN-BOTTOM: 0px"&gt;
 &lt;font style="FONT-FAMILY: Times New Roman" size="2"&gt;For the three
 and six months ended June&amp;#xA0;30, 2013 we incurred approximately
 $2.2 million and $4.3 million in exploration expenses,
 respectively, as compared to $1.2 million and $2.3 million in
 exploration expenses for the same periods ended June&amp;#xA0;30, 2012,
 respectively. Approximately $3.8 million of the expense incurred in
 2013 was due to geological and geophysical type expenditures and
 delay rental payments. An additional $0.5 million was related to
 one exploratory well in Posey County, Indiana determined to be a
 dry hole. Approximately $0.8 million of the expense incurred in
 2012 was due to geological and geophysical type expenditures and
 delay rental payments primarily associated with leases in the
 Appalachian Basin. An additional $0.3 million was related to the
 plugging of two exploratory Marcellus Shale wells that were spud
 during 2011 in Butler County, Pennsylvania. Minimal drilling was
 completed on these wells before a strategic decision was made to
 abandon the well sites and defer capital to other leases in the
 development plan and hold the acreage by production.&lt;/font&gt;&lt;/p&gt;
 &lt;/div&gt;</NonNumbericText><FootnoteIndexer /><CurrencyCode /><CurrencySymbol /><IsIndependantCurrency>false</IsIndependantCurrency><ShowCurrencySymbol>false</ShowCurrencySymbol><DisplayDateInUSFormat>false</DisplayDateInUSFormat></Cell></Cells><ElementDataType>nonnum:textBlockItemType</ElementDataType><SimpleDataType>na</SimpleDataType><ElementDefenition>Exploration expense.</ElementDefenition><ElementReferences>No definition available.</ElementReferences><IsTotalLabel>false</IsTotalLabel><UnitID>0</UnitID><Label>Exploration Expense</Label></Row></Rows><Footnotes /><IsEquityReport>false</IsEquityReport><ReportName>Exploration Expense</ReportName><MonetaryRoundingLevel>UnKnown</MonetaryRoundingLevel><SharesRoundingLevel>UnKnown</SharesRoundingLevel><PerShareRoundingLevel>UnKnown</PerShareRoundingLevel><ExchangeRateRoundingLevel>UnKnown</ExchangeRateRoundingLevel><HasCustomUnits>true</HasCustomUnits><IsEmbedReport>false</IsEmbedReport><IsMultiCurrency>false</IsMultiCurrency><ReportType>Sheet</ReportType><RoleURI>http://www.rexenergycorp.com/taxonomy/role/NotesToFinancialStatementsExplorationExpenseDisclosureTextBlock</RoleURI><NumberOfCols>1</NumberOfCols><NumberOfRows>1</NumberOfRows></InstanceReport>
